From b6e372108d293d8af76185c475b592b10dac012e Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Thu, 23 Oct 2014 16:31:57 +0900 Subject: [PATCH 01/29] TAJO-1123: Use Fragment instead of FileFragment. --- .../org/apache/tajo/jdbc/TajoResultSet.java | 7 +- .../java/org/apache/tajo/TajoConstants.java | 1 + .../engine/planner/PhysicalPlannerImpl.java | 59 +- .../tajo/engine/planner/PlannerUtil.java | 9 +- .../planner/physical/BSTIndexScanExec.java | 7 +- .../physical/ColPartitionStoreExec.java | 3 +- .../planner/physical/ExternalSortExec.java | 14 +- .../physical/HashShuffleFileWriteExec.java | 2 +- .../physical/PartitionMergeScanExec.java | 7 +- .../physical/RangeShuffleFileWriteExec.java | 6 +- .../engine/planner/physical/SeqScanExec.java | 16 +- .../planner/physical/StoreTableExec.java | 5 +- .../tajo/engine/query/QueryUnitRequest.java | 3 +- .../engine/query/QueryUnitRequestImpl.java | 4 +- .../DefaultFragmentScheduleAlgorithm.java | 6 +- .../tajo/master/DefaultTaskScheduler.java | 5 +- .../org/apache/tajo/master/FragmentPair.java | 14 +- .../org/apache/tajo/master/GlobalEngine.java | 69 +- .../GreedyFragmentScheduleAlgorithm.java | 11 +- .../apache/tajo/master/LazyTaskScheduler.java | 21 +- .../master/NonForwardQueryResultScanner.java | 3 +- .../org/apache/tajo/master/TajoMaster.java | 7 +- .../tajo/master/TajoMasterClientService.java | 3 +- .../master/event/FragmentScheduleEvent.java | 15 +- .../apache/tajo/master/querymaster/Query.java | 6 +- .../tajo/master/querymaster/QueryMaster.java | 9 - .../master/querymaster/QueryMasterTask.java | 4 - .../tajo/master/querymaster/QueryUnit.java | 14 +- .../master/querymaster/Repartitioner.java | 66 +- .../tajo/master/querymaster/SubQuery.java | 36 +- .../java/org/apache/tajo/util/IndexUtil.java | 2 +- .../apache/tajo/worker/TajoQueryEngine.java | 4 +- .../java/org/apache/tajo/worker/Task.java | 2 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../planner/global/TestBroadcastJoinPlan.java | 2 +- .../planner/physical/TestBNLJoinExec.java | 18 +- .../planner/physical/TestBSTIndexExec.java | 16 +- .../physical/TestExternalSortExec.java | 13 +- .../physical/TestFullOuterHashJoinExec.java | 36 +- .../physical/TestFullOuterMergeJoinExec.java | 53 +- .../physical/TestHashAntiJoinExec.java | 12 +- .../planner/physical/TestHashJoinExec.java | 18 +- .../physical/TestHashSemiJoinExec.java | 12 +- .../physical/TestLeftOuterHashJoinExec.java | 40 +- .../physical/TestLeftOuterNLJoinExec.java | 43 +- .../planner/physical/TestMergeJoinExec.java | 14 +- .../planner/physical/TestNLJoinExec.java | 18 +- .../planner/physical/TestPhysicalPlanner.java | 123 +-- .../TestProgressExternalSortExec.java | 11 +- .../physical/TestRightOuterHashJoinExec.java | 26 +- .../physical/TestRightOuterMergeJoinExec.java | 48 +- .../engine/planner/physical/TestSortExec.java | 10 +- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../tajo/engine/query/TestJoinQuery.java | 71 +- .../org/apache/tajo/jdbc/TestResultSet.java | 6 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../apache/tajo/storage/TestFileFragment.java | 4 +- .../org/apache/tajo/storage/TestRowFile.java | 7 +- .../worker/TestRangeRetrieverHandler.java | 14 +- .../java/org/apache/tajo/storage/CSVFile.java | 8 +- .../org/apache/tajo/storage/FileScanner.java | 7 +- .../tajo/storage/FileStorageManager.java | 702 ++++++++++++++++++ .../storage/HashShuffleAppenderManager.java | 4 +- .../org/apache/tajo/storage/MergeScanner.java | 23 +- .../org/apache/tajo/storage/NullScanner.java | 4 +- .../java/org/apache/tajo/storage/RawFile.java | 24 +- .../java/org/apache/tajo/storage/RowFile.java | 8 +- .../apache/tajo/storage/StorageManager.java | 634 ++-------------- .../apache/tajo/storage/avro/AvroScanner.java | 3 +- .../tajo/storage/fragment/FileFragment.java | 17 +- .../tajo/storage/fragment/Fragment.java | 8 + .../tajo/storage/parquet/ParquetScanner.java | 4 +- .../apache/tajo/storage/rcfile/RCFile.java | 9 +- .../sequencefile/SequenceFileScanner.java | 6 +- .../tajo/storage/trevni/TrevniScanner.java | 6 +- .../tajo/storage/TestCompressionStorages.java | 10 +- .../apache/tajo/storage/TestFileSystems.java | 11 +- .../apache/tajo/storage/TestMergeScanner.java | 11 +- .../tajo/storage/TestStorageManager.java | 23 +- .../org/apache/tajo/storage/TestStorages.java | 58 +- .../tajo/storage/index/TestBSTIndex.java | 20 +- .../index/TestSingleCSVFileBSTIndex.java | 4 +- 82 files changed, 1414 insertions(+), 1251 deletions(-) create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java index 65954f1fc8..aa6d1554e5 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java @@ -34,6 +34,7 @@ import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; import java.sql.SQLException; @@ -91,7 +92,7 @@ private void initScanner() throws IOException { totalRow = INFINITE_ROW_NUM; } - List frags = getFragments(desc.getPath()); + List frags = getFragments(desc.getPath()); scanner = new MergeScanner(conf, desc.getSchema(), desc.getMeta(), frags); } } @@ -110,9 +111,9 @@ public int compare(FileStatus f1, FileStatus f2) { } } - private List getFragments(Path tablePath) + private List getFragments(Path tablePath) throws IOException { - List fragments = Lists.newArrayList(); + List fragments = Lists.newArrayList(); FileStatus[] files = fs.listStatus(tablePath, new PathFilter() { @Override public boolean accept(Path path) { diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java index be617830ed..08909b4f46 100644 --- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java @@ -38,6 +38,7 @@ public class TajoConstants { public static final String SYSTEM_HA_ACTIVE_DIR_NAME = "active"; public static final String SYSTEM_HA_BACKUP_DIR_NAME = "backup"; + public static final int UNKNOWN_ROW_NUMBER = -1; private TajoConstants() {} } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 485677a41d..d4c5b70acb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -45,10 +45,12 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.IndexUtil; @@ -56,7 +58,6 @@ import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; import java.util.Stack; @@ -75,11 +76,9 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { private static final int UNGENERATED_PID = -1; protected final TajoConf conf; - protected final StorageManager sm; - public PhysicalPlannerImpl(final TajoConf conf, final StorageManager sm) { + public PhysicalPlannerImpl(final TajoConf conf) { this.conf = conf; - this.sm = sm; } public PhysicalExec createPlan(final TaskAttemptContext context, final LogicalNode logicalPlan) @@ -252,7 +251,7 @@ public long estimateSizeRecursive(TaskAttemptContext ctx, String [] tableIds) th List fragments = FragmentConvertor.convert(ctx.getConf(), CatalogProtos.StoreType.CSV, ctx.getTables(tableId)); for (FileFragment frag : fragments) { - size += frag.getEndKey(); + size += frag.getLength(); } } return size; @@ -444,13 +443,13 @@ private MergeJoinExec createMergeInnerJoin(TaskAttemptContext context, JoinNode leftSortNode.setSortSpecs(sortSpecs[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); - ExternalSortExec outerSort = new ExternalSortExec(context, sm, leftSortNode, leftExec); + ExternalSortExec outerSort = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); - ExternalSortExec innerSort = new ExternalSortExec(context, sm, rightSortNode, rightExec); + ExternalSortExec innerSort = new ExternalSortExec(context, rightSortNode, rightExec); LOG.info("Join (" + plan.getPID() +") chooses [Merge Join]"); return new MergeJoinExec(context, plan, outerSort, innerSort, sortSpecs[0], sortSpecs[1]); @@ -541,13 +540,13 @@ private PhysicalExec createRightOuterMergeJoinPlan(TaskAttemptContext context, J leftSortNode2.setSortSpecs(sortSpecs2[0]); leftSortNode2.setInSchema(leftExec.getSchema()); leftSortNode2.setOutSchema(leftExec.getSchema()); - ExternalSortExec outerSort2 = new ExternalSortExec(context, sm, leftSortNode2, leftExec); + ExternalSortExec outerSort2 = new ExternalSortExec(context, leftSortNode2, leftExec); SortNode rightSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode2.setSortSpecs(sortSpecs2[1]); rightSortNode2.setInSchema(rightExec.getSchema()); rightSortNode2.setOutSchema(rightExec.getSchema()); - ExternalSortExec innerSort2 = new ExternalSortExec(context, sm, rightSortNode2, rightExec); + ExternalSortExec innerSort2 = new ExternalSortExec(context, rightSortNode2, rightExec); return new RightOuterMergeJoinExec(context, plan, outerSort2, innerSort2, sortSpecs2[0], sortSpecs2[1]); } @@ -632,13 +631,13 @@ private MergeFullOuterJoinExec createFullOuterMergeJoinPlan(TaskAttemptContext c leftSortNode.setSortSpecs(sortSpecs3[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); - ExternalSortExec outerSort3 = new ExternalSortExec(context, sm, leftSortNode, leftExec); + ExternalSortExec outerSort3 = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs3[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); - ExternalSortExec innerSort3 = new ExternalSortExec(context, sm, rightSortNode, rightExec); + ExternalSortExec innerSort3 = new ExternalSortExec(context, rightSortNode, rightExec); return new MergeFullOuterJoinExec(context, plan, outerSort3, innerSort3, sortSpecs3[0], sortSpecs3[1]); } @@ -766,7 +765,7 @@ public PhysicalExec createShuffleFileWritePlan(TaskAttemptContext ctx, switch (plan.getShuffleType()) { case HASH_SHUFFLE: case SCATTERED_HASH_SHUFFLE: - return new HashShuffleFileWriteExec(ctx, sm, plan, subOp); + return new HashShuffleFileWriteExec(ctx, plan, subOp); case RANGE_SHUFFLE: SortExec sortExec = PhysicalPlanUtil.findExecutor(subOp, SortExec.class); @@ -781,7 +780,7 @@ public PhysicalExec createShuffleFileWritePlan(TaskAttemptContext ctx, specs[i] = new SortSpec(columns[i]); } } - return new RangeShuffleFileWriteExec(ctx, sm, subOp, plan.getInSchema(), plan.getInSchema(), sortSpecs); + return new RangeShuffleFileWriteExec(ctx, subOp, plan.getInSchema(), plan.getInSchema(), sortSpecs); case NONE_SHUFFLE: // if there is no given NULL CHAR property in the table property and the query is neither CTAS or INSERT, @@ -867,7 +866,7 @@ private PhysicalExec createSortBasedColumnPartitionStorePlan(TaskAttemptContext sortNode.setInSchema(child.getSchema()); sortNode.setOutSchema(child.getSchema()); - ExternalSortExec sortExec = new ExternalSortExec(context, sm, sortNode, child); + ExternalSortExec sortExec = new ExternalSortExec(context, sortNode, child); LOG.info("The planner chooses [Sort-based Column Partitioned Store] algorithm"); return new SortBasedColPartitionStoreExec(context, storeTableNode, sortExec); } @@ -894,10 +893,10 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St // Since the default intermediate file format is raw file, it is not problem right now. if (checkIfSortEquivalance(ctx, scanNode, node)) { if (ctx.getTable(scanNode.getCanonicalName()) == null) { - return new SeqScanExec(ctx, sm, scanNode, null); + return new SeqScanExec(ctx, scanNode, null); } FragmentProto [] fragments = ctx.getTables(scanNode.getCanonicalName()); - return new ExternalSortExec(ctx, sm, (SortNode) node.peek(), fragments); + return new ExternalSortExec(ctx, (SortNode) node.peek(), fragments); } else { Enforcer enforcer = ctx.getEnforcer(); @@ -917,25 +916,26 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St if (scanNode instanceof PartitionedTableScanNode) { if (broadcastFlag) { PartitionedTableScanNode partitionedTableScanNode = (PartitionedTableScanNode) scanNode; - List fileFragments = TUtil.newList(); + List fileFragments = TUtil.newList(); + FileStorageManager fileStorageManager = StorageManager.getFileStorageManager(ctx.getConf()); for (Path path : partitionedTableScanNode.getInputPaths()) { - fileFragments.addAll(TUtil.newList(sm.split(scanNode.getCanonicalName(), path))); + fileFragments.addAll(TUtil.newList(fileStorageManager.split(scanNode.getCanonicalName(), path))); } FragmentProto[] fragments = FragmentConvertor.toFragmentProtoArray(fileFragments.toArray(new FileFragment[fileFragments.size()])); ctx.addFragments(scanNode.getCanonicalName(), fragments); - return new PartitionMergeScanExec(ctx, sm, scanNode, fragments); + return new PartitionMergeScanExec(ctx, scanNode, fragments); } } } if (ctx.getTable(scanNode.getCanonicalName()) == null) { - return new SeqScanExec(ctx, sm, scanNode, null); + return new SeqScanExec(ctx, scanNode, null); } FragmentProto [] fragments = ctx.getTables(scanNode.getCanonicalName()); - return new SeqScanExec(ctx, sm, scanNode, fragments); + return new SeqScanExec(ctx, scanNode, fragments); } } @@ -995,7 +995,7 @@ private PhysicalExec createSortAggregation(TaskAttemptContext ctx, EnforceProper sortNode.setSortSpecs(sortSpecs); sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(subOp.getSchema()); - ExternalSortExec sortExec = new ExternalSortExec(ctx, sm, sortNode, subOp); + ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp); LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")"); return new SortAggregateExec(ctx, groupbyNode, sortExec); } @@ -1036,7 +1036,7 @@ public PhysicalExec createWindowAgg(TaskAttemptContext context,WindowAggNode win sortNode.setSortSpecs(sortSpecs); sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(subOp.getSchema()); - child = new ExternalSortExec(context, sm, sortNode, subOp); + child = new ExternalSortExec(context, sortNode, subOp); LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")"); } @@ -1099,7 +1099,7 @@ private SortExec createSortExecForDistinctGroupby(TaskAttemptContext context, sortNode.setSortSpecs(sortSpecs.toArray(new SortSpec[]{})); sortNode.setInSchema(distinctNode.getInSchema()); sortNode.setOutSchema(distinctNode.getInSchema()); - ExternalSortExec sortExec = new ExternalSortExec(context, sm, sortNode, subOp); + ExternalSortExec sortExec = new ExternalSortExec(context, sortNode, subOp); return sortExec; } @@ -1130,7 +1130,7 @@ private PhysicalExec createSortAggregationDistinctGroupbyExec(TaskAttemptContext sortNode.setSortSpecs(sortSpecs); sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(eachGroupbyNode.getInSchema()); - ExternalSortExec sortExec = new ExternalSortExec(ctx, sm, sortNode, subOp); + ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp); sortAggregateExec[index++] = new SortAggregateExec(ctx, eachGroupbyNode, sortExec); } @@ -1158,7 +1158,7 @@ public PhysicalExec createSortPlan(TaskAttemptContext context, SortNode sortNode if (algorithm == SortEnforce.SortAlgorithm.IN_MEMORY_SORT) { return new MemSortExec(context, sortNode, child); } else { - return new ExternalSortExec(context, sm, sortNode, child); + return new ExternalSortExec(context, sortNode, child); } } @@ -1167,7 +1167,7 @@ public PhysicalExec createSortPlan(TaskAttemptContext context, SortNode sortNode public SortExec createBestSortPlan(TaskAttemptContext context, SortNode sortNode, PhysicalExec child) throws IOException { - return new ExternalSortExec(context, sm, sortNode, child); + return new ExternalSortExec(context, sortNode, child); } public PhysicalExec createIndexScanExec(TaskAttemptContext ctx, @@ -1182,11 +1182,12 @@ public PhysicalExec createIndexScanExec(TaskAttemptContext ctx, FragmentConvertor.convert(ctx.getConf(), ctx.getDataChannel().getStoreType(), fragmentProtos); String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys()); - Path indexPath = new Path(sm.getTablePath(annotation.getTableName()), "index"); + Path indexPath = new Path( + StorageManager.getFileStorageManager(ctx.getConf()).getTablePath(annotation.getTableName()), "index"); TupleComparator comp = new TupleComparator(annotation.getKeySchema(), annotation.getSortKeys()); - return new BSTIndexScanExec(ctx, sm, annotation, fragments.get(0), new Path(indexPath, indexName), + return new BSTIndexScanExec(ctx, annotation, fragments.get(0), new Path(indexPath, indexName), annotation.getKeySchema(), comp, annotation.getDatum()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java index 827be83d1d..02e5c2de71 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java @@ -32,12 +32,14 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.exception.InvalidQueryException; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; @@ -178,6 +180,11 @@ public static Collection getRelationLineageWithinQueryBlock(LogicalPlan return visitor.getFoundRelations(); } + public static boolean isFileStorageType(StoreType storageType) { + //Currently all storage type are a file storage. + return true; + } + public static class RelationFinderVisitor extends BasicLogicalPlanVisitor { private Set foundRelNameSet = Sets.newHashSet(); @@ -855,7 +862,7 @@ private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List 0) { for (FileStatus eachFile : files) { if (result.size() >= numResultFiles) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 51f70a0b7e..018df5f23b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.physical; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.tajo.catalog.Schema; @@ -47,8 +48,7 @@ public class BSTIndexScanExec extends PhysicalExec { private float progress; - public BSTIndexScanExec(TaskAttemptContext context, - StorageManager sm , ScanNode scanNode , + public BSTIndexScanExec(TaskAttemptContext context, ScanNode scanNode , FileFragment fragment, Path fileName , Schema keySchema, TupleComparator comparator , Datum[] datum) throws IOException { super(context, scanNode.getInSchema(), scanNode.getOutSchema()); @@ -61,7 +61,8 @@ public BSTIndexScanExec(TaskAttemptContext context, this.fileScanner.init(); this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets()); - this.reader = new BSTIndex(sm.getFileSystem().getConf()). + FileSystem fs = fileName.getFileSystem(context.getConf()); + this.reader = new BSTIndex(fs.getConf()). getIndexReader(fileName, keySchema, comparator); this.reader.open(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index a07290401c..b28b648f0b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -160,7 +160,8 @@ public void openAppender(int suffixId) throws IOException { actualFilePath = new Path(lastFileName + "_" + suffixId); } - appender = StorageManager.getStorageManager(context.getConf()).getAppender(meta, outSchema, actualFilePath); + appender = StorageManager.getFileStorageManager(context.getConf()) + .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); appender.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java index cfd7fb784d..bd6d1141a4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java @@ -106,7 +106,7 @@ public class ExternalSortExec extends SortExec { /** total bytes of input data */ private long sortAndStoredBytes; - private ExternalSortExec(final TaskAttemptContext context, final StorageManager sm, final SortNode plan) + private ExternalSortExec(final TaskAttemptContext context, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); @@ -128,10 +128,9 @@ private ExternalSortExec(final TaskAttemptContext context, final StorageManager localFS = new RawLocalFileSystem(); } - public ExternalSortExec(final TaskAttemptContext context, - final StorageManager sm, final SortNode plan, + public ExternalSortExec(final TaskAttemptContext context,final SortNode plan, final CatalogProtos.FragmentProto[] fragments) throws PhysicalPlanningException { - this(context, sm, plan); + this(context, plan); mergedInputFragments = TUtil.newList(); for (CatalogProtos.FragmentProto proto : fragments) { @@ -140,10 +139,9 @@ public ExternalSortExec(final TaskAttemptContext context, } } - public ExternalSortExec(final TaskAttemptContext context, - final StorageManager sm, final SortNode plan, final PhysicalExec child) + public ExternalSortExec(final TaskAttemptContext context, final SortNode plan, final PhysicalExec child) throws IOException { - this(context, sm, plan); + this(context, plan); setChild(child); } @@ -777,7 +775,7 @@ public void close() throws IOException { if (finalOutputFiles != null) { for (FileFragment frag : finalOutputFiles) { File tmpFile = new File(localFS.makeQualified(frag.getPath()).toUri()); - if (frag.getStartKey() == 0 && frag.getEndKey() == tmpFile.length()) { + if (frag.getStartKey() == 0 && frag.getLength() == tmpFile.length()) { localFS.delete(frag.getPath(), true); LOG.info("Delete file: " + frag); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java index 47fcb8d71f..f30ab79b02 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java @@ -55,7 +55,7 @@ public final class HashShuffleFileWriteExec extends UnaryPhysicalExec { private HashShuffleAppenderManager hashShuffleAppenderManager; private int numHashShuffleBufferTuples; - public HashShuffleFileWriteExec(TaskAttemptContext context, final StorageManager sm, + public HashShuffleFileWriteExec(TaskAttemptContext context, final ShuffleFileWriteNode plan, final PhysicalExec child) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema(), child); Preconditions.checkArgument(plan.hasShuffleKeys()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java index 4c72075c79..80af870cc1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java @@ -44,25 +44,22 @@ public class PartitionMergeScanExec extends PhysicalExec { private List scanners = Lists.newArrayList(); private Iterator iterator; - private StorageManager sm; - private float progress; protected TableStats inputStats; - public PartitionMergeScanExec(TaskAttemptContext context, StorageManager sm, + public PartitionMergeScanExec(TaskAttemptContext context, ScanNode plan, CatalogProtos.FragmentProto[] fragments) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema()); this.plan = plan; this.fragments = fragments; - this.sm = sm; inputStats = new TableStats(); } public void init() throws IOException { for (CatalogProtos.FragmentProto fragment : fragments) { - SeqScanExec scanExec = new SeqScanExec(context, sm, (ScanNode) PlannerUtil.clone(null, plan), + SeqScanExec scanExec = new SeqScanExec(context, (ScanNode) PlannerUtil.clone(null, plan), new CatalogProtos.FragmentProto[] {fragment}); scanners.add(scanExec); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index 4e5d1cfac2..fb693a7e3f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -51,7 +51,7 @@ public class RangeShuffleFileWriteExec extends UnaryPhysicalExec { private FileAppender appender; private TableMeta meta; - public RangeShuffleFileWriteExec(final TaskAttemptContext context, final StorageManager sm, + public RangeShuffleFileWriteExec(final TaskAttemptContext context, final PhysicalExec child, final Schema inSchema, final Schema outSchema, final SortSpec[] sortSpecs) throws IOException { super(context, inSchema, outSchema, child); @@ -70,7 +70,7 @@ public void init() throws IOException { indexKeys[i] = inSchema.getColumnId(col.getQualifiedName()); } - BSTIndex bst = new BSTIndex(new TajoConf()); + BSTIndex bst = new BSTIndex(new TajoConf(context.getConf())); this.comp = new TupleComparator(keySchema, sortSpecs); Path storeTablePath = new Path(context.getWorkDir(), "output"); LOG.info("Output data directory: " + storeTablePath); @@ -78,7 +78,7 @@ public void init() throws IOException { context.getDataChannel().getStoreType() : CatalogProtos.StoreType.RAW); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); - this.appender = (FileAppender) StorageManager.getStorageManager(context.getConf()).getAppender(meta, + this.appender = (FileAppender) StorageManager.getFileStorageManager(context.getConf()).getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(); this.appender.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index c7f8e2dd9d..f881a007f2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -40,6 +40,7 @@ import org.apache.tajo.engine.utils.TupleUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.worker.TaskAttemptContext; @@ -67,7 +68,7 @@ public class SeqScanExec extends PhysicalExec { private boolean cacheRead = false; - public SeqScanExec(TaskAttemptContext context, StorageManager sm, ScanNode plan, + public SeqScanExec(TaskAttemptContext context, ScanNode plan, CatalogProtos.FragmentProto [] fragments) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema()); @@ -79,9 +80,9 @@ public SeqScanExec(TaskAttemptContext context, StorageManager sm, ScanNode plan, String pathNameKey = ""; if (fragments != null) { for (FragmentProto f : fragments) { - FileFragment fileFragement = FragmentConvertor.convert( + Fragment fragement = FragmentConvertor.convert( context.getConf(), plan.getTableDesc().getMeta().getStoreType(), f); - pathNameKey += fileFragement.getPath(); + pathNameKey += fragement.getKey(); } } @@ -214,13 +215,14 @@ private void initScanner(Schema projected) throws IOException { if (fragments != null) { if (fragments.length > 1) { this.scanner = new MergeScanner(context.getConf(), plan.getPhysicalSchema(), plan.getTableDesc().getMeta(), - FragmentConvertor.convert(context.getConf(), plan.getTableDesc().getMeta().getStoreType(), + FragmentConvertor.convert(context.getConf(), plan.getTableDesc().getMeta().getStoreType(), fragments), projected ); } else { - this.scanner = StorageManager.getStorageManager( - context.getConf()).getScanner(plan.getTableDesc().getMeta(), plan.getPhysicalSchema(), fragments[0], - projected); + StorageManager storageManager = StorageManager.getStorageManager( + context.getConf(), plan.getTableDesc().getMeta().getStoreType()); + this.scanner = storageManager.getScanner(plan.getTableDesc().getMeta(), + plan.getPhysicalSchema(), fragments[0], projected); } scanner.init(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index b496e42c9d..af5e11c49b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -89,10 +89,11 @@ public void openNewFile(int suffixId) throws IOException { if (plan instanceof InsertNode) { InsertNode createTableNode = (InsertNode) plan; - appender = StorageManager.getStorageManager(context.getConf()).getAppender(meta, + appender = StorageManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender(meta, createTableNode.getTableSchema(), context.getOutputPath()); } else { - appender = StorageManager.getStorageManager(context.getConf()).getAppender(meta, outSchema, lastFileName); + appender = StorageManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender(meta, + outSchema, lastFileName); } appender.enableStats(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java index dc9a63d934..731f64d1b4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java @@ -24,6 +24,7 @@ import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -44,7 +45,7 @@ public interface QueryUnitRequest extends ProtoObject getFetches(); public boolean shouldDie(); public void setShouldDie(); - public QueryContext getQueryContext(); + public QueryContext getQueryContext(TajoConf baseConf); public DataChannel getDataChannel(); public Enforcer getEnforcer(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java index ef82427b43..c874b70e8e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java @@ -186,7 +186,7 @@ public void addFetch(String name, FetchImpl fetch) { fetches.add(fetch); } - public QueryContext getQueryContext() { + public QueryContext getQueryContext(TajoConf baseConf) { QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder; if (queryContext != null) { return queryContext; @@ -194,7 +194,7 @@ public QueryContext getQueryContext() { if (!p.hasQueryContext()) { return null; } - this.queryContext = new QueryContext(new TajoConf(), p.getQueryContext()); + this.queryContext = new QueryContext(new TajoConf(baseConf), p.getQueryContext()); return this.queryContext; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java b/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java index e4b98d4d4d..6a2a705478 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java @@ -21,6 +21,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.util.RackResolver; +import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.NetUtils; import java.util.*; @@ -81,7 +82,10 @@ public boolean isEmpty() { @Override public void addFragment(FragmentPair fragmentPair) { String[] hosts = fragmentPair.getLeftFragment().getHosts(); - int[] diskIds = fragmentPair.getLeftFragment().getDiskIds(); + int[] diskIds = null; + if (fragmentPair.getLeftFragment() instanceof FileFragment) { + diskIds = ((FileFragment)fragmentPair.getLeftFragment()).getDiskIds(); + } for (int i = 0; i < hosts.length; i++) { addFragment(hosts[i], diskIds[i], fragmentPair); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java index 62d4892a4e..b20c5d806a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java @@ -43,6 +43,7 @@ import org.apache.tajo.master.querymaster.SubQuery; import org.apache.tajo.storage.DataLocation; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; import org.apache.tajo.worker.FetchImpl; @@ -149,8 +150,8 @@ public void stop() { super.stop(); } - private FileFragment[] fragmentsForNonLeafTask; - private FileFragment[] broadcastFragmentsForNonLeafTask; + private Fragment[] fragmentsForNonLeafTask; + private Fragment[] broadcastFragmentsForNonLeafTask; LinkedList taskRequestEvents = new LinkedList(); public void schedule() { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/FragmentPair.java b/tajo-core/src/main/java/org/apache/tajo/master/FragmentPair.java index 598b1c5aa9..827386b643 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/FragmentPair.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/FragmentPair.java @@ -19,7 +19,7 @@ package org.apache.tajo.master; import com.google.common.base.Objects; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; /** * FragmentPair consists of two fragments, a left fragment and a right fragment. @@ -29,23 +29,23 @@ * For other queries, it is assumed to have only a left fragment. */ public class FragmentPair { - private FileFragment leftFragment; - private FileFragment rightFragment; + private Fragment leftFragment; + private Fragment rightFragment; - public FragmentPair(FileFragment left) { + public FragmentPair(Fragment left) { this.leftFragment = left; } - public FragmentPair(FileFragment left, FileFragment right) { + public FragmentPair(Fragment left, Fragment right) { this.leftFragment = left; this.rightFragment = right; } - public FileFragment getLeftFragment() { + public Fragment getLeftFragment() { return leftFragment; } - public FileFragment getRightFragment() { + public Fragment getRightFragment() { return rightFragment; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 382b789207..2bcd301970 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -38,6 +38,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes; @@ -685,32 +686,18 @@ private TableDesc createTable(QueryContext queryContext, CreateTableNode createT meta = CatalogUtil.newTableMeta(createTable.getStorageType()); } - if(createTable.isExternal()){ + if(PlannerUtil.isFileStorageType(createTable.getStorageType()) && createTable.isExternal()){ Preconditions.checkState(createTable.hasPath(), "ERROR: LOCATION must be given."); - } else { - String databaseName; - String tableName; - if (CatalogUtil.isFQTableName(createTable.getTableName())) { - databaseName = CatalogUtil.extractQualifier(createTable.getTableName()); - tableName = CatalogUtil.extractSimpleName(createTable.getTableName()); - } else { - databaseName = queryContext.getCurrentDatabase(); - tableName = createTable.getTableName(); - } - - // create a table directory (i.e., ${WAREHOUSE_DIR}/${DATABASE_NAME}/${TABLE_NAME} ) - Path tablePath = StorageUtil.concatPath(sm.getWarehouseDir(), databaseName, tableName); - createTable.setPath(tablePath); } - return createTableOnPath(queryContext, createTable.getTableName(), createTable.getTableSchema(), - meta, createTable.getPath(), createTable.isExternal(), createTable.getPartitionMethod(), ifNotExists); + return createTable(queryContext, createTable.getTableName(), createTable.getStorageType(), + createTable.getTableSchema(), meta, createTable.getPath(), createTable.isExternal(), + createTable.getPartitionMethod(), ifNotExists); } - public TableDesc createTableOnPath(QueryContext queryContext, String tableName, Schema schema, TableMeta meta, - Path path, boolean isExternal, PartitionMethodDesc partitionDesc, - boolean ifNotExists) - throws IOException { + public TableDesc createTable(QueryContext queryContext, String tableName, StoreType storeType, + Schema schema, TableMeta meta, Path location, boolean isExternal, + PartitionMethodDesc partitionDesc, boolean ifNotExists) throws IOException { String databaseName; String simpleTableName; if (CatalogUtil.isFQTableName(tableName)) { @@ -734,39 +721,15 @@ public TableDesc createTableOnPath(QueryContext queryContext, String tableName, } } - FileSystem fs = path.getFileSystem(context.getConf()); - - if (isExternal) { - if(!fs.exists(path)) { - LOG.error("ERROR: " + path.toUri() + " does not exist"); - throw new IOException("ERROR: " + path.toUri() + " does not exist"); - } - } else { - fs.mkdirs(path); - } - - long totalSize = 0; - - try { - totalSize = sm.calculateSize(path); - } catch (IOException e) { - LOG.warn("Cannot calculate the size of the relation", e); - } - - TableStats stats = new TableStats(); - stats.setNumBytes(totalSize); - - if (isExternal) { // if it is an external table, there is no way to know the exact row number without processing. - stats.setNumRows(TajoClient.UNKNOWN_ROW_NUMBER); - } - TableDesc desc = new TableDesc(CatalogUtil.buildFQName(databaseName, simpleTableName), - schema, meta, path, isExternal); - desc.setStats(stats); + schema, meta, location, isExternal); + if (partitionDesc != null) { desc.setPartitionMethod(partitionDesc); } + StorageManager.getStorageManager(queryContext.getConf(), storeType).createTable(desc); + if (catalog.createTable(desc)) { LOG.info("Table " + desc.getName() + " is created (" + desc.getStats().getNumBytes() + ")"); return desc; @@ -860,15 +823,15 @@ public boolean dropTable(QueryContext queryContext, String tableName, boolean if } } - Path path = catalog.getTableDesc(qualifiedName).getPath(); + TableDesc tableDesc = catalog.getTableDesc(qualifiedName); catalog.dropTable(qualifiedName); if (purge) { try { - FileSystem fs = path.getFileSystem(context.getConf()); - fs.delete(path, true); + StorageManager.getStorageManager(queryContext.getConf(), tableDesc.getMeta().getStoreType()) + .purgeTable(tableDesc); } catch (IOException e) { - throw new InternalError(e.getMessage()); + LOG.warn("Can't purge table: " + e.getMessage(), e); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java b/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java index 82fd6fce7b..37983996af 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.util.RackResolver; import org.apache.tajo.master.DefaultFragmentScheduleAlgorithm.FragmentsPerDisk; +import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.NetUtils; import org.apache.tajo.util.TUtil; @@ -101,7 +102,10 @@ private void updateHostPriority(HostAndDisk hostAndDisk, int priority) { @Override public void addFragment(FragmentPair fragmentPair) { String[] hosts = fragmentPair.getLeftFragment().getHosts(); - int[] diskIds = fragmentPair.getLeftFragment().getDiskIds(); + int[] diskIds = null; + if (fragmentPair.getLeftFragment() instanceof FileFragment) { + diskIds = ((FileFragment)fragmentPair.getLeftFragment()).getDiskIds(); + } for (int i = 0; i < hosts.length; i++) { addFragment(hosts[i], diskIds[i], fragmentPair); } @@ -276,7 +280,10 @@ public FragmentPair[] getAllFragments() { public void removeFragment(FragmentPair fragmentPair) { String [] hosts = fragmentPair.getLeftFragment().getHosts(); - int[] diskIds = fragmentPair.getLeftFragment().getDiskIds(); + int[] diskIds = null; + if (fragmentPair.getLeftFragment() instanceof FileFragment) { + diskIds = ((FileFragment)fragmentPair.getLeftFragment()).getDiskIds(); + } for (int i = 0; i < hosts.length; i++) { String normalizedHost = NetUtils.normalizeHost(hosts[i]); Map diskFragmentMap = fragmentHostMapping.get(normalizedHost); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java index f7953e02c1..50a118e00b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java @@ -39,6 +39,7 @@ import org.apache.tajo.master.querymaster.QueryUnitAttempt; import org.apache.tajo.master.querymaster.SubQuery; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; import org.apache.tajo.worker.FetchImpl; @@ -197,15 +198,17 @@ public void handle(TaskSchedulerEvent event) { if (event.getType() == EventType.T_SCHEDULE) { if (event instanceof FragmentScheduleEvent) { FragmentScheduleEvent castEvent = (FragmentScheduleEvent) event; - Collection rightFragments = castEvent.getRightFragments(); + Collection rightFragments = castEvent.getRightFragments(); if (rightFragments == null || rightFragments.isEmpty()) { scheduledFragments.addFragment(new FragmentPair(castEvent.getLeftFragment(), null)); } else { - for (FileFragment eachFragment: rightFragments) { + for (Fragment eachFragment: rightFragments) { scheduledFragments.addFragment(new FragmentPair(castEvent.getLeftFragment(), eachFragment)); } } - initDiskBalancer(castEvent.getLeftFragment().getHosts(), castEvent.getLeftFragment().getDiskIds()); + if (castEvent.getLeftFragment() instanceof FileFragment) { + initDiskBalancer(castEvent.getLeftFragment().getHosts(), ((FileFragment)castEvent.getLeftFragment()).getDiskIds()); + } } else if (event instanceof FetchScheduleEvent) { FetchScheduleEvent castEvent = (FetchScheduleEvent) event; scheduledFetches.addFetch(castEvent.getFetches()); @@ -375,13 +378,13 @@ private void assignLeafTasks(List taskRequests) { break; } - if (assignedFragmentSize + fragmentPair.getLeftFragment().getEndKey() > taskSize) { + if (assignedFragmentSize + fragmentPair.getLeftFragment().getLength() > taskSize) { break; } else { fragmentPairs.add(fragmentPair); - assignedFragmentSize += fragmentPair.getLeftFragment().getEndKey(); + assignedFragmentSize += fragmentPair.getLeftFragment().getLength(); if (fragmentPair.getRightFragment() != null) { - assignedFragmentSize += fragmentPair.getRightFragment().getEndKey(); + assignedFragmentSize += fragmentPair.getRightFragment().getLength(); } } scheduledFragments.removeFragment(fragmentPair); @@ -397,13 +400,13 @@ private void assignLeafTasks(List taskRequests) { break; } - if (assignedFragmentSize + fragmentPair.getLeftFragment().getEndKey() > taskSize) { + if (assignedFragmentSize + fragmentPair.getLeftFragment().getLength() > taskSize) { break; } else { fragmentPairs.add(fragmentPair); - assignedFragmentSize += fragmentPair.getLeftFragment().getEndKey(); + assignedFragmentSize += fragmentPair.getLeftFragment().getLength(); if (fragmentPair.getRightFragment() != null) { - assignedFragmentSize += fragmentPair.getRightFragment().getEndKey(); + assignedFragmentSize += fragmentPair.getRightFragment().getLength(); } } scheduledFragments.removeFragment(fragmentPair); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java index c0bd8420ba..41ba4749aa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java @@ -87,8 +87,7 @@ private void initSeqScanExec() throws IOException { try { // scanNode must be clone cause SeqScanExec change target in the case of a partitioned table. - scanExec = new SeqScanExec(taskContext, - StorageManager.getStorageManager(tajoConf), (ScanNode)scanNode.clone(), fragments); + scanExec = new SeqScanExec(taskContext, (ScanNode)scanNode.clone(), fragments); } catch (CloneNotSupportedException e) { throw new IOException(e.getMessage(), e); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 8d4a41f8b7..e47eb6012f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -49,6 +49,7 @@ import org.apache.tajo.master.rm.WorkerResourceManager; import org.apache.tajo.master.session.SessionManager; import org.apache.tajo.rpc.RpcChannelFactory; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.*; import org.apache.tajo.util.metrics.TajoSystemMetrics; @@ -104,7 +105,7 @@ public class TajoMaster extends CompositeService { private CatalogServer catalogServer; private CatalogService catalog; - private StorageManager storeManager; + private FileStorageManager storeManager; private GlobalEngine globalEngine; private AsyncDispatcher dispatcher; private TajoMasterClientService tajoMasterClientService; @@ -159,7 +160,7 @@ public void serviceInit(Configuration _conf) throws Exception { // check the system directory and create if they are not created. checkAndInitializeSystemDirectories(); - this.storeManager = StorageManager.getStorageManager(systemConf); + this.storeManager = StorageManager.getFileStorageManager(systemConf, null); catalogServer = new CatalogServer(FunctionLoader.load()); addIfService(catalogServer); @@ -394,7 +395,7 @@ public CatalogServer getCatalogServer() { return this.catalogServer; } - public StorageManager getStorageManager() { + public FileStorageManager getStorageManager() { return this.storeManager; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index a4688d9950..8968c6eba8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -762,7 +762,8 @@ public TableResponse createExternalTable(RpcController controller, CreateTableRe TableDesc desc; try { - desc = context.getGlobalEngine().createTableOnPath(queryContext, request.getName(), schema, + desc = context.getGlobalEngine().createTable(queryContext, request.getName(), + meta.getStoreType(), schema, meta, path, true, partitionDesc, false); } catch (Exception e) { return TableResponse.newBuilder() diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/FragmentScheduleEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/FragmentScheduleEvent.java index 8cc17cb458..c38f99fb73 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/event/FragmentScheduleEvent.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/event/FragmentScheduleEvent.java @@ -20,22 +20,23 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.util.Collection; public class FragmentScheduleEvent extends TaskSchedulerEvent { - private final FileFragment leftFragment; - private final Collection rightFragments; + private final Fragment leftFragment; + private final Collection rightFragments; public FragmentScheduleEvent(final EventType eventType, final ExecutionBlockId blockId, - final FileFragment fragment) { + final Fragment fragment) { this(eventType, blockId, fragment, null); } public FragmentScheduleEvent(final EventType eventType, final ExecutionBlockId blockId, - final FileFragment leftFragment, - final Collection rightFragments) { + final Fragment leftFragment, + final Collection rightFragments) { super(eventType, blockId); this.leftFragment = leftFragment; this.rightFragments = rightFragments; @@ -45,11 +46,11 @@ public boolean hasRightFragments() { return this.rightFragments != null && !this.rightFragments.isEmpty(); } - public FileFragment getLeftFragment() { + public Fragment getLeftFragment() { return leftFragment; } - public Collection getRightFragments() { return rightFragments; } + public Collection getRightFragments() { return rightFragments; } @Override public String toString() { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index b8240b832c..2d61c846e9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -70,7 +70,6 @@ public class Query implements EventHandler { private Map subqueries; private final EventHandler eventHandler; private final MasterPlan plan; - private final StorageManager sm; QueryMasterTask.QueryMasterTaskContext context; private ExecutionBlockCursor cursor; @@ -212,7 +211,6 @@ public Query(final QueryMasterTask.QueryMasterTaskContext context, final QueryId subqueries = Maps.newHashMap(); this.eventHandler = eventHandler; this.plan = plan; - this.sm = context.getStorageManager(); this.cursor = new ExecutionBlockCursor(plan, true); StringBuilder sb = new StringBuilder("\n======================================================="); @@ -358,7 +356,7 @@ public void transition(Query query, QueryEvent queryEvent) { query.setStartTime(); SubQuery subQuery = new SubQuery(query.context, query.getPlan(), - query.getExecutionBlockCursor().nextBlock(), query.sm); + query.getExecutionBlockCursor().nextBlock()); subQuery.setPriority(query.priority--); query.addSubQuery(subQuery); @@ -864,7 +862,7 @@ private boolean hasNext(Query query) { private void executeNextBlock(Query query) { ExecutionBlockCursor cursor = query.getExecutionBlockCursor(); ExecutionBlock nextBlock = cursor.nextBlock(); - SubQuery nextSubQuery = new SubQuery(query.context, query.getPlan(), nextBlock, query.sm); + SubQuery nextSubQuery = new SubQuery(query.context, query.getPlan(), nextBlock); nextSubQuery.setPriority(query.priority--); query.addSubQuery(nextSubQuery); nextSubQuery.handle(new SubQueryEvent(nextSubQuery.getId(), SubQueryEventType.SQ_INIT)); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java index 7c3d799a9a..235d827aa9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java @@ -41,7 +41,6 @@ import org.apache.tajo.rpc.NullCallback; import org.apache.tajo.rpc.RpcConnectionPool; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; -import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.HAServiceUtil; import org.apache.tajo.util.NetUtils; import org.apache.tajo.worker.TajoWorker; @@ -70,8 +69,6 @@ public class QueryMaster extends CompositeService implements EventHandler { private GlobalPlanner globalPlanner; - private StorageManager storageManager; - private TajoConf systemConf; private Map queryMasterTasks = Maps.newConcurrentMap(); @@ -115,8 +112,6 @@ public void init(Configuration conf) { this.dispatcher = new TajoAsyncDispatcher("querymaster_" + System.currentTimeMillis()); addIfService(dispatcher); - this.storageManager = StorageManager.getStorageManager(systemConf); - globalPlanner = new GlobalPlanner(systemConf, workerContext); dispatcher.register(QueryStartEvent.EventType.class, new QueryStartEventHandler()); @@ -372,10 +367,6 @@ public Clock getClock() { return clock; } - public StorageManager getStorageManager() { - return storageManager; - } - public QueryMaster getQueryMaster() { return QueryMaster.this; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index cb06df940b..dd994bc69f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -556,10 +556,6 @@ public QueryId getQueryId() { return queryId; } - public StorageManager getStorageManager() { - return queryMasterContext.getStorageManager(); - } - public Path getStagingDir() { return queryContext.getStagingDir(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java index 8a3ef7487f..a769089f25 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java @@ -42,6 +42,7 @@ import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext; import org.apache.tajo.storage.DataLocation; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TajoIdUtils; import org.apache.tajo.worker.FetchImpl; @@ -237,15 +238,18 @@ public void setLogicalPlan(LogicalNode plan) { } } - private void addDataLocation(FileFragment fragment) { + private void addDataLocation(Fragment fragment) { String[] hosts = fragment.getHosts(); - int[] diskIds = fragment.getDiskIds(); + int[] diskIds = null; + if (fragment instanceof FileFragment) { + diskIds = ((FileFragment)fragment).getDiskIds(); + } for (int i = 0; i < hosts.length; i++) { dataLocations.add(new DataLocation(hosts[i], diskIds[i])); } } - public void addFragment(FileFragment fragment, boolean useDataLocation) { + public void addFragment(Fragment fragment, boolean useDataLocation) { Set fragmentProtos; if (fragMap.containsKey(fragment.getTableName())) { fragmentProtos = fragMap.get(fragment.getTableName()); @@ -260,8 +264,8 @@ public void addFragment(FileFragment fragment, boolean useDataLocation) { totalFragmentNum++; } - public void addFragments(Collection fragments) { - for (FileFragment eachFragment: fragments) { + public void addFragments(Collection fragments) { + for (Fragment eachFragment: fragments) { addFragment(eachFragment, false); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 615ebcfcb5..a2b21932bd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -47,10 +47,12 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty; import org.apache.tajo.master.TaskSchedulerContext; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Pair; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.TUtil; @@ -81,18 +83,19 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC MasterPlan masterPlan = subQuery.getMasterPlan(); ExecutionBlock execBlock = subQuery.getBlock(); QueryMasterTask.QueryMasterTaskContext masterContext = subQuery.getContext(); - StorageManager storageManager = subQuery.getStorageManager(); ScanNode[] scans = execBlock.getScanNodes(); Path tablePath; - FileFragment[] fragments = new FileFragment[scans.length]; + Fragment[] fragments = new Fragment[scans.length]; long[] stats = new long[scans.length]; // initialize variables from the child operators for (int i = 0; i < scans.length; i++) { TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName()); if (tableDesc == null) { // if it is a real table stored on storage + FileStorageManager storageManager = StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + tablePath = storageManager.getTablePath(scans[i].getTableName()); if (execBlock.getUnionScanMap() != null && !execBlock.getUnionScanMap().isEmpty()) { for (Map.Entry unionScanEntry: execBlock.getUnionScanMap().entrySet()) { @@ -105,21 +108,23 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC } fragments[i] = new FileFragment(scans[i].getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); } else { - tablePath = tableDesc.getPath(); try { stats[i] = GlobalPlanner.computeDescendentVolume(scans[i]); } catch (PlanningException e) { throw new IOException(e); } + StorageManager storageManager = + StorageManager.getStorageManager(subQuery.getContext().getConf(), tableDesc.getMeta().getStoreType()); + // if table has no data, storageManager will return empty FileFragment. // So, we need to handle FileFragment by its size. // If we don't check its size, it can cause IndexOutOfBoundsException. - List fileFragments = storageManager.getSplits(scans[i].getCanonicalName(), tableDesc.getMeta(), tableDesc.getSchema(), tablePath); + List fileFragments = storageManager.getSplits(scans[i].getCanonicalName(), tableDesc); if (fileFragments.size() > 0) { fragments[i] = fileFragments.get(0); } else { - fragments[i] = new FileFragment(scans[i].getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); + fragments[i] = new FileFragment(scans[i].getCanonicalName(), tableDesc.getPath(), 0, 0, new String[]{UNKNOWN_HOST}); } } } @@ -266,14 +271,14 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC //select intermediate scan and stats ScanNode[] intermediateScans = new ScanNode[largeScanIndexList.size()]; long[] intermediateScanStats = new long[largeScanIndexList.size()]; - FileFragment[] intermediateFragments = new FileFragment[largeScanIndexList.size()]; + Fragment[] intermediateFragments = new Fragment[largeScanIndexList.size()]; int index = 0; for (Integer eachIdx : largeScanIndexList) { intermediateScans[index] = scans[eachIdx]; intermediateScanStats[index] = stats[eachIdx]; intermediateFragments[index++] = fragments[eachIdx]; } - FileFragment[] broadcastFragments = new FileFragment[broadcastIndexList.size()]; + Fragment[] broadcastFragments = new Fragment[broadcastIndexList.size()]; ScanNode[] broadcastScans = new ScanNode[broadcastIndexList.size()]; index = 0; for (Integer eachIdx : broadcastIndexList) { @@ -307,9 +312,9 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster SubQuery subQuery, ScanNode[] scans, long[] stats, - FileFragment[] fragments, + Fragment[] fragments, ScanNode[] broadcastScans, - FileFragment[] broadcastFragments) throws IOException { + Fragment[] broadcastFragments) throws IOException { MasterPlan masterPlan = subQuery.getMasterPlan(); ExecutionBlock execBlock = subQuery.getBlock(); // The hash map is modeling as follows: @@ -392,7 +397,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster int joinTaskNum = Math.min(maxTaskNum, hashEntries.size()); LOG.info("The determined number of join tasks is " + joinTaskNum); - List rightFragments = new ArrayList(); + List rightFragments = new ArrayList(); rightFragments.add(fragments[1]); if (broadcastFragments != null) { @@ -402,14 +407,17 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster Path[] partitionScanPaths = null; TableDesc tableDesc = masterContext.getTableDescMap().get(eachScan.getCanonicalName()); if (eachScan.getType() == NodeType.PARTITIONS_SCAN) { + FileStorageManager storageManager = StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)eachScan; partitionScanPaths = partitionScan.getInputPaths(); // set null to inputPaths in getFragmentsFromPartitionedTable() - getFragmentsFromPartitionedTable(subQuery.getStorageManager(), eachScan, tableDesc); + getFragmentsFromPartitionedTable(storageManager, eachScan, tableDesc); partitionScan.setInputPaths(partitionScanPaths); } else { - Collection scanFragments = subQuery.getStorageManager().getSplits(eachScan.getCanonicalName(), - tableDesc.getMeta(), tableDesc.getSchema(), tableDesc.getPath()); + StorageManager storageManager = StorageManager.getStorageManager(subQuery.getContext().getConf(), + tableDesc.getMeta().getStoreType()); + Collection scanFragments = storageManager.getSplits(eachScan.getCanonicalName(), tableDesc); if (scanFragments != null) { rightFragments.addAll(scanFragments); } @@ -478,10 +486,10 @@ public static Map>> merge /** * It creates a number of fragments for all partitions. */ - public static List getFragmentsFromPartitionedTable(StorageManager sm, + public static List getFragmentsFromPartitionedTable(FileStorageManager sm, ScanNode scan, TableDesc table) throws IOException { - List fragments = Lists.newArrayList(); + List fragments = Lists.newArrayList(); PartitionedTableScanNode partitionsScan = (PartitionedTableScanNode) scan; fragments.addAll(sm.getSplits( scan.getCanonicalName(), table.getMeta(), table.getSchema(), partitionsScan.getInputPaths())); @@ -490,7 +498,7 @@ public static List getFragmentsFromPartitionedTable(StorageManager } private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext schedulerContext, SubQuery subQuery, - int baseScanId, FileFragment[] fragments) throws IOException { + int baseScanId, Fragment[] fragments) throws IOException { ExecutionBlock execBlock = subQuery.getBlock(); ScanNode[] scans = execBlock.getScanNodes(); @@ -509,23 +517,27 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch // . add all partition paths to node's inputPaths variable // -> SCAN // . add all fragments to broadcastFragments - Collection baseFragments = null; - List broadcastFragments = new ArrayList(); + Collection baseFragments = null; + List broadcastFragments = new ArrayList(); for (int i = 0; i < scans.length; i++) { ScanNode scan = scans[i]; TableDesc desc = subQuery.getContext().getTableDescMap().get(scan.getCanonicalName()); TableMeta meta = desc.getMeta(); - Collection scanFragments; + Collection scanFragments; Path[] partitionScanPaths = null; if (scan.getType() == NodeType.PARTITIONS_SCAN) { PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)scan; partitionScanPaths = partitionScan.getInputPaths(); // set null to inputPaths in getFragmentsFromPartitionedTable() - scanFragments = getFragmentsFromPartitionedTable(subQuery.getStorageManager(), scan, desc); + FileStorageManager storageManager = + StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + scanFragments = getFragmentsFromPartitionedTable(storageManager, scan, desc); } else { - scanFragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, desc.getSchema(), - desc.getPath()); + StorageManager storageManager = + StorageManager.getStorageManager(subQuery.getContext().getConf(), desc.getMeta().getStoreType()); + + scanFragments = storageManager.getSplits(scan.getCanonicalName(), desc); } if (scanFragments != null) { @@ -628,7 +640,7 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo ExecutionBlock execBlock = subQuery.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName()); + tablePath = StorageManager.getFileStorageManager(subQuery.getContext().getConf()).getTablePath(scan.getTableName()); ExecutionBlock sampleChildBlock = masterPlan.getChild(subQuery.getId(), 0); SortNode sortNode = PlannerUtil.findTopNode(sampleChildBlock.getPlan(), NodeType.SORT); @@ -770,14 +782,14 @@ public long getVolume() { public static void scheduleHashShuffledFetches(TaskSchedulerContext schedulerContext, MasterPlan masterPlan, SubQuery subQuery, DataChannel channel, - int maxNum) { + int maxNum) throws IOException { ExecutionBlock execBlock = subQuery.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName()); + tablePath = StorageManager.getFileStorageManager(subQuery.getContext().getConf()).getTablePath(scan.getTableName()); - FileFragment frag = new FileFragment(scan.getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); - List fragments = new ArrayList(); + Fragment frag = new FileFragment(scan.getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); + List fragments = new ArrayList(); fragments.add(frag); SubQuery.scheduleFragments(subQuery, fragments); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index a126144c50..0dd32d49b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -61,8 +61,10 @@ import org.apache.tajo.master.event.*; import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.worker.FetchImpl; @@ -94,7 +96,6 @@ public class SubQuery implements EventHandler { private TableStats resultStatistics; private TableStats inputStatistics; private EventHandler eventHandler; - private final StorageManager sm; private AbstractTaskScheduler taskScheduler; private QueryMasterTask.QueryMasterTaskContext context; private final List diagnostics = new ArrayList(); @@ -282,12 +283,10 @@ SubQueryEventType.SQ_KILL, new KillTasksTransition()) private List hashShuffleIntermediateEntries = new ArrayList(); private AtomicInteger completeReportReceived = new AtomicInteger(0); - public SubQuery(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, - ExecutionBlock block, StorageManager sm) { + public SubQuery(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, ExecutionBlock block) { this.context = context; this.masterPlan = masterPlan; this.block = block; - this.sm = sm; this.eventHandler = context.getEventHandler(); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); @@ -435,10 +434,6 @@ public int getPriority() { return this.priority; } - public StorageManager getStorageManager() { - return sm; - } - public ExecutionBlockId getId() { return block.getId(); } @@ -976,7 +971,7 @@ private static void scheduleFragmentsForLeafQuery(SubQuery subQuery) throws IOEx ScanNode scan = scans[0]; TableDesc table = subQuery.context.getTableDescMap().get(scan.getCanonicalName()); - Collection fragments; + Collection fragments; TableMeta meta = table.getMeta(); // Depending on scanner node's type, it creates fragments. If scan is for @@ -985,10 +980,13 @@ private static void scheduleFragmentsForLeafQuery(SubQuery subQuery) throws IOEx // span a number of blocks or possibly consists of a number of files. if (scan.getType() == NodeType.PARTITIONS_SCAN) { // After calling this method, partition paths are removed from the physical plan. - fragments = Repartitioner.getFragmentsFromPartitionedTable(subQuery.getStorageManager(), scan, table); + FileStorageManager storageManager = + StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + fragments = Repartitioner.getFragmentsFromPartitionedTable(storageManager, scan, table); } else { - Path inputPath = table.getPath(); - fragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, table.getSchema(), inputPath); + StorageManager storageManager = + StorageManager.getStorageManager(subQuery.getContext().getConf(), meta.getStoreType()); + fragments = storageManager.getSplits(scan.getCanonicalName(), table); } SubQuery.scheduleFragments(subQuery, fragments); @@ -1006,27 +1004,27 @@ private static void scheduleFragmentsForLeafQuery(SubQuery subQuery) throws IOEx } } - public static void scheduleFragment(SubQuery subQuery, FileFragment fragment) { + public static void scheduleFragment(SubQuery subQuery, Fragment fragment) { subQuery.taskScheduler.handle(new FragmentScheduleEvent(TaskSchedulerEvent.EventType.T_SCHEDULE, subQuery.getId(), fragment)); } - public static void scheduleFragments(SubQuery subQuery, Collection fragments) { - for (FileFragment eachFragment : fragments) { + public static void scheduleFragments(SubQuery subQuery, Collection fragments) { + for (Fragment eachFragment : fragments) { scheduleFragment(subQuery, eachFragment); } } - public static void scheduleFragments(SubQuery subQuery, Collection leftFragments, - Collection broadcastFragments) { - for (FileFragment eachLeafFragment : leftFragments) { + public static void scheduleFragments(SubQuery subQuery, Collection leftFragments, + Collection broadcastFragments) { + for (Fragment eachLeafFragment : leftFragments) { scheduleFragment(subQuery, eachLeafFragment, broadcastFragments); } } public static void scheduleFragment(SubQuery subQuery, - FileFragment leftFragment, Collection rightFragments) { + Fragment leftFragment, Collection rightFragments) { subQuery.taskScheduler.handle(new FragmentScheduleEvent(TaskSchedulerEvent.EventType.T_SCHEDULE, subQuery.getId(), leftFragment, rightFragments)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java index ad8f120861..af65f5fb65 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java @@ -38,7 +38,7 @@ public class IndexUtil { public static String getIndexNameOfFrag(FileFragment fragment, SortSpec[] keys) { StringBuilder builder = new StringBuilder(); builder.append(fragment.getPath().getName() + "_"); - builder.append(fragment.getStartKey() + "_" + fragment.getEndKey() + "_"); + builder.append(fragment.getStartKey() + "_" + fragment.getLength() + "_"); for(int i = 0 ; i < keys.length ; i ++) { builder.append(keys[i].getSortKey().getSimpleName()+"_"); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java index 5cf6c4619c..81886fdf31 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java @@ -30,12 +30,10 @@ public class TajoQueryEngine { - private final StorageManager storageManager; private final PhysicalPlanner phyPlanner; public TajoQueryEngine(TajoConf conf) throws IOException { - this.storageManager = StorageManager.getStorageManager(conf); - this.phyPlanner = new PhysicalPlannerImpl(conf, storageManager); + this.phyPlanner = new PhysicalPlannerImpl(conf); } public PhysicalExec createPlan(TaskAttemptContext ctx, LogicalNode plan) diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index a7eaaf852d..b3025c05c9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -148,7 +148,7 @@ public Task(String taskRunnerId, this.taskId = taskId; this.systemConf = executionBlockContext.getConf(); - this.queryContext = request.getQueryContext(); + this.queryContext = request.getQueryContext(systemConf); this.executionBlockContext = executionBlockContext; this.taskDir = StorageUtil.concatPath(baseDir, taskId.getQueryUnitId().getId() + "_" + taskId.getId()); diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index 45d3c5109e..0959f2ead1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -47,7 +47,7 @@ public class BackendTestingUtil { public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { - StorageManager sm = StorageManager.getStorageManager(conf, tablePath); + FileStorageManager sm = StorageManager.getFileStorageManager(conf, tablePath); FileSystem fs = sm.getFileSystem(); Appender appender; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 8810692be1..99e894c904 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -141,7 +141,7 @@ private TableDesc makeTestData(String tableName, Schema schema, int dataSize) th contentsData += j; } } - Appender appender = StorageManager.getStorageManager(conf).getAppender(tableMeta, schema, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(tableMeta, schema, dataPath); appender.init(); Tuple tuple = new VTuple(schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 197ff653e1..404e6adaae 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -76,7 +76,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -86,7 +86,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < OUTER_TUPLE_NUM; i++) { @@ -107,7 +107,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < INNER_TUPLE_NUM; i += 2) { @@ -147,9 +147,9 @@ public final void testBNLCrossJoin() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin"); @@ -157,7 +157,7 @@ public final void testBNLCrossJoin() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -178,9 +178,9 @@ public final void testBNLInnerJoin() throws IOException, PlanningException { LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context).getRootBlock().getRoot(); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -196,7 +196,7 @@ public final void testBNLInnerJoin() throws IOException, PlanningException { ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 00a44165cb..99c616d606 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -66,7 +66,7 @@ public class TestBSTIndexExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private StorageManager sm; + private FileStorageManager sm; private Schema idxSchema; private TupleComparator comp; private BSTIndex.BSTIndexWriter writer; @@ -91,7 +91,7 @@ public void setup() throws Exception { Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = StorageManager.getStorageManager(conf, workDir); + sm = StorageManager.getFileStorageManager(conf, workDir); idxPath = new Path(workDir, "test.idx"); @@ -117,7 +117,7 @@ public void setup() throws Exception { fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); - FileAppender appender = (FileAppender)StorageManager.getStorageManager(conf).getAppender(meta, schema, + FileAppender appender = (FileAppender)StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -167,7 +167,7 @@ public void testEqual() throws Exception { this.rndKey = rnd.nextInt(250); final String QUERY = "select * from employee where managerId = " + rndKey; - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); @@ -175,7 +175,7 @@ public void testEqual() throws Exception { LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); - TmpPlanner phyPlanner = new TmpPlanner(conf, sm); + TmpPlanner phyPlanner = new TmpPlanner(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); int tupleCount = this.randomValues.get(rndKey); @@ -189,8 +189,8 @@ public void testEqual() throws Exception { } private class TmpPlanner extends PhysicalPlannerImpl { - public TmpPlanner(TajoConf conf, StorageManager sm) { - super(conf, sm); + public TmpPlanner(TajoConf conf) { + super(conf); } @Override @@ -204,7 +204,7 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St Datum[] datum = new Datum[]{DatumFactory.createInt4(rndKey)}; - return new BSTIndexScanExec(ctx, sm, scanNode, fragments.get(0), idxPath, idxSchema, comp , datum); + return new BSTIndexScanExec(ctx, scanNode, fragments.get(0), idxPath, idxSchema, comp , datum); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 6572506b25..111772c654 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -56,7 +56,7 @@ public class TestExternalSortExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; + private FileStorageManager sm; private Path testDir; private final int numTuple = 100000; @@ -73,7 +73,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -82,7 +82,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -118,7 +118,7 @@ public void tearDown() throws Exception { @Test public final void testNext() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -128,7 +128,7 @@ public final void testNext() throws IOException, PlanningException { LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = plan.getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); ProjectionExec proj = (ProjectionExec) exec; @@ -138,8 +138,7 @@ public final void testNext() throws IOException, PlanningException { UnaryPhysicalExec sortExec = proj.getChild(); SeqScanExec scan = sortExec.getChild(); - ExternalSortExec extSort = new ExternalSortExec(ctx, sm, - ((MemSortExec)sortExec).getPlan(), scan); + ExternalSortExec extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan); proj.setChild(extSort); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index aba5169585..2907d11923 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -58,7 +58,7 @@ public class TestFullOuterHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; + private FileStorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -81,7 +81,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -104,7 +104,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -133,7 +133,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -172,7 +172,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -224,7 +224,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); @@ -263,8 +263,8 @@ public final void testFullOuterHashJoinExec0() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0"); @@ -272,7 +272,7 @@ public final void testFullOuterHashJoinExec0() throws IOException, PlanningExcep LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -300,8 +300,8 @@ public final void testFullOuterHashJoinExec1() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1"); @@ -309,7 +309,7 @@ public final void testFullOuterHashJoinExec1() throws IOException, PlanningExcep LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -336,8 +336,8 @@ public final void testFullOuterHashJoinExec2() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2"); @@ -345,7 +345,7 @@ public final void testFullOuterHashJoinExec2() throws IOException, PlanningExcep LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -373,8 +373,8 @@ public final void testFullOuterHashJoinExec3() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -384,7 +384,7 @@ public final void testFullOuterHashJoinExec3() throws IOException, PlanningExcep workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 7998c61bbf..df014b0ce2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -85,7 +85,7 @@ public void setUp() throws Exception { catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -108,7 +108,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -146,7 +146,7 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManager.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = StorageManager.getFileStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -177,7 +177,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -216,7 +216,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -268,7 +268,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); @@ -311,9 +311,9 @@ public final void testFullOuterMergeJoin0() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] dep3Frags = - StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0"); @@ -321,7 +321,7 @@ public final void testFullOuterMergeJoin0() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -348,9 +348,9 @@ public final void testFullOuterMergeJoin1() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] job3Frags = - StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1"); @@ -358,7 +358,7 @@ public final void testFullOuterMergeJoin1() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -385,9 +385,9 @@ public final void testFullOuterMergeJoin2() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] job3Frags = - StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2"); @@ -395,7 +395,7 @@ public final void testFullOuterMergeJoin2() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -423,9 +423,9 @@ public final void testFullOuterMergeJoin3() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] dep4Frags = - StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), dep4.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), dep4.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3"); @@ -433,7 +433,7 @@ public final void testFullOuterMergeJoin3() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -463,9 +463,9 @@ public final void testFullOuterMergeJoin4() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] phone3Frags = - StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -474,7 +474,7 @@ public final void testFullOuterMergeJoin4() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -502,9 +502,9 @@ public final void testFullOuterMergeJoin5() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] phone3Frags = - StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); @@ -513,7 +513,7 @@ public final void testFullOuterMergeJoin5() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -530,9 +530,4 @@ public final void testFullOuterMergeJoin5() throws IOException, PlanningExceptio exec.close(); assertEquals(7, count); } - - - - - } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 77f524d30d..17c1da0f9f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -72,7 +72,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -82,7 +82,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { @@ -146,9 +146,9 @@ public void tearDown() throws Exception { @Test public final void testHashAntiJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -162,7 +162,7 @@ public final void testHashAntiJoin() throws IOException, PlanningException { optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); // replace an equal join with an hash anti join. diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index eff647d9f1..93b6547065 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -75,7 +75,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -85,7 +85,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { @@ -149,8 +149,8 @@ public final void testHashInnerJoin() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); @@ -158,7 +158,7 @@ public final void testHashInnerJoin() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -190,9 +190,9 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -202,7 +202,7 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P ctx.setEnforcer(enforcer); ctx.getQueryContext().setLong(SessionVars.HASH_JOIN_SIZE_LIMIT.keyname(), 100l); - PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 148592dea2..0f27d080be 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -72,7 +72,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -82,7 +82,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); // make 27 tuples @@ -150,9 +150,9 @@ public void tearDown() throws Exception { @Test public final void testHashSemiJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -166,7 +166,7 @@ public final void testHashSemiJoin() throws IOException, PlanningException { optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); // replace an equal join with an hash anti join. diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 3e404859db..506c5bca92 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -82,7 +82,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -105,7 +105,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -134,7 +134,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -173,7 +173,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -225,7 +225,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); @@ -267,8 +267,8 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0"); @@ -276,7 +276,7 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -295,9 +295,9 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep @Test public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningException { - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); @@ -309,7 +309,7 @@ public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningExce Expr expr = analyzer.parse(QUERIES[1]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -336,9 +336,9 @@ public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningExce @Test public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -350,7 +350,7 @@ public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningExce Expr expr = analyzer.parse(QUERIES[2]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -378,9 +378,9 @@ public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningExce @Test public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -392,7 +392,7 @@ public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningExce Expr expr = analyzer.parse(QUERIES[3]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -420,9 +420,9 @@ public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningExce @Test public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, "default.emp3", emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, "default.emp3", emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, "default.phone3", phone3.getMeta(), phone3.getPath(), + FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, "default.phone3", phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); @@ -434,7 +434,7 @@ public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningExce Expr expr = analyzer.parse(QUERIES[4]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 9ee51c3d5b..fa26c94e87 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -81,7 +81,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -104,7 +104,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -133,7 +133,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -172,7 +172,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -224,7 +224,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); @@ -254,9 +254,9 @@ public void tearDown() throws Exception { @Test public final void testLeftOuterNLJoinExec0() throws IOException, PlanningException { - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), + FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); @@ -269,7 +269,7 @@ public final void testLeftOuterNLJoinExec0() throws IOException, PlanningExcepti LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); //maybe plan results with hash join exec algorithm usage. Must convert from HashLeftOuterJoinExec into NLLeftOuterJoinExec @@ -295,9 +295,9 @@ public final void testLeftOuterNLJoinExec0() throws IOException, PlanningExcepti @Test public final void testLeftOuterNLJoinExec1() throws IOException, PlanningException { - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); @@ -311,7 +311,7 @@ public final void testLeftOuterNLJoinExec1() throws IOException, PlanningExcepti LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); //maybe plan results with hash join exec algorithm usage. Must convert from HashLeftOuterJoinExec into NLLeftOuterJoinExec @@ -340,9 +340,9 @@ public final void testLeftOuterNLJoinExec1() throws IOException, PlanningExcepti @Test public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -355,7 +355,7 @@ public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningExcept LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); //maybe plan results with hash join exec algorithm usage. Must convert from HashLeftOuterJoinExec into NLLeftOuterJoinExec @@ -385,9 +385,9 @@ public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningExcept @Test public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -400,7 +400,7 @@ public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningExcept LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); //maybe plan results with hash join exec algorithm usage. Must convert from HashLeftOuterJoinExec into NLLeftOuterJoinExec @@ -429,9 +429,9 @@ public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningExcept @Test public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); @@ -444,7 +444,7 @@ public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningExcept LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); //maybe plan results with hash join exec algorithm usage. Must convert from HashLeftOuterJoinExec into NLLeftOuterJoinExec @@ -470,7 +470,4 @@ public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningExcept exec.close(); assertEquals(0, count); } - - - } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index ddad0a21bf..8270b38224 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -75,7 +75,7 @@ public void setUp() throws Exception { catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); FileSystem fs = testDir.getFileSystem(conf); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -85,7 +85,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -114,7 +114,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { @@ -161,8 +161,10 @@ public final void testMergeInnerJoin() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); - FileFragment[] empFrags = sm.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = sm.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", + employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin"); @@ -170,7 +172,7 @@ public final void testMergeInnerJoin() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, root); ProjectionExec proj = (ProjectionExec) exec; assertTrue(proj.getChild() instanceof MergeJoinExec); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 20053cbf30..2273e4ce76 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -76,7 +76,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -86,7 +86,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 50; i++) { @@ -109,7 +109,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 50; i += 2) { @@ -144,9 +144,9 @@ public void tearDown() throws Exception { @Test public final void testNLCrossJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -159,7 +159,7 @@ public final void testNLCrossJoin() throws IOException, PlanningException { LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); int i = 0; @@ -173,9 +173,9 @@ public final void testNLCrossJoin() throws IOException, PlanningException { @Test public final void testNLInnerJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -188,7 +188,7 @@ public final void testNLInnerJoin() throws IOException, PlanningException { LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); Tuple tuple; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 30c2cd5e38..b332a89383 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -51,6 +51,7 @@ import org.apache.tajo.storage.*; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.index.bst.BSTIndex; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.CommonTestingUtil; @@ -84,7 +85,7 @@ public class TestPhysicalPlanner { private static SQLAnalyzer analyzer; private static LogicalPlanner planner; private static LogicalOptimizer optimizer; - private static StorageManager sm; + private static FileStorageManager sm; private static Path testDir; private static Session session = LocalTajoTestingUtility.createDummySession(); private static QueryContext defaultContext; @@ -102,7 +103,7 @@ public static void setUp() throws Exception { util.startCatalogCluster(); conf = util.getConfiguration(); testDir = CommonTestingUtil.getTestDir("target/test-data/TestPhysicalPlanner"); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); @@ -125,7 +126,7 @@ public static void setUp() throws Exception { Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -144,7 +145,7 @@ public static void setUp() throws Exception { Path scorePath = new Path(testDir, "score"); TableMeta scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV, new KeyValueSet()); - appender = StorageManager.getStorageManager(conf).getAppender(scoreMeta, scoreSchema, scorePath); + appender = StorageManager.getFileStorageManager(conf).getAppender(scoreMeta, scoreSchema, scorePath); appender.init(); score = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta, @@ -185,7 +186,7 @@ public static void createLargeScoreTable() throws IOException { Schema scoreSchmea = score.getSchema(); TableMeta scoreLargeMeta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - Appender appender = StorageManager.getStorageManager(conf).getAppender(scoreLargeMeta, scoreSchmea, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init(); @@ -242,7 +243,7 @@ public static void tearDown() throws Exception { @Test public final void testCreateScanPlan() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -255,7 +256,7 @@ public final void testCreateScanPlan() throws IOException, PlanningException { optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); Tuple tuple; @@ -273,7 +274,7 @@ public final void testCreateScanPlan() throws IOException, PlanningException { @Test public final void testCreateScanWithFilterPlan() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanWithFilterPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -286,7 +287,7 @@ public final void testCreateScanWithFilterPlan() throws IOException, PlanningExc optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); Tuple tuple; @@ -302,7 +303,7 @@ public final void testCreateScanWithFilterPlan() throws IOException, PlanningExc @Test public final void testGroupByPlan() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -314,7 +315,7 @@ public final void testGroupByPlan() throws IOException, PlanningException { optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); int i = 0; @@ -333,7 +334,7 @@ public final void testGroupByPlan() throws IOException, PlanningException { @Test public final void testHashGroupByPlanWithALLField() throws IOException, PlanningException { // TODO - currently, this query does not use hash-based group operator. - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir( "target/test-data/testHashGroupByPlanWithALLField"); @@ -345,7 +346,7 @@ public final void testHashGroupByPlanWithALLField() throws IOException, Planning LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); int i = 0; @@ -363,7 +364,7 @@ public final void testHashGroupByPlanWithALLField() throws IOException, Planning @Test public final void testSortGroupByPlan() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortGroupByPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -374,7 +375,7 @@ public final void testSortGroupByPlan() throws IOException, PlanningException { LogicalPlan plan = planner.createPlan(defaultContext, context); optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan.getRootBlock().getRoot()); /*HashAggregateExec hashAgg = (HashAggregateExec) exec; @@ -426,7 +427,7 @@ public final void testSortGroupByPlan() throws IOException, PlanningException { @Test public final void testStorePlan() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -442,13 +443,13 @@ public final void testStorePlan() throws IOException, PlanningException { TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.CSV); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); exec.close(); - Scanner scanner = StorageManager.getStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), + Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -473,7 +474,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann TableStats stats = largeScore.getStats(); assertTrue("Checking meaningfulness of test", stats.getNumBytes() > StorageUnit.MB); - FileFragment[] frags = StorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(), largeScore.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithMaxOutputFileSize"); @@ -493,7 +494,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann LogicalNode rootNode = optimizer.optimize(plan); // executing StoreTableExec - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); @@ -508,7 +509,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann // checking the file contents long totalNum = 0; for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { - Scanner scanner = StorageManager.getStorageManager(conf).getFileScanner( + Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner( CatalogUtil.newTableMeta(StoreType.CSV), rootNode.getOutSchema(), status.getPath()); @@ -524,7 +525,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann @Test public final void testStorePlanWithRCFile() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithRCFile"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -539,13 +540,13 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.RCFILE); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); exec.close(); - Scanner scanner = StorageManager.getStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), + Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -565,7 +566,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio @Test public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -577,7 +578,7 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep Expr context = analyzer.parse(CreateTableAsStmts[2]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); assertTrue(exec instanceof SortBasedColPartitionStoreExec); } @@ -592,7 +593,7 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc Enforcer enforcer = new Enforcer(); enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.HASH_PARTITION); - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -601,7 +602,7 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc ctx.setEnforcer(enforcer); ctx.setOutputPath(new Path(workDir, "grouped4")); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); assertTrue(exec instanceof HashBasedColPartitionStoreExec); } @@ -616,7 +617,7 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc Enforcer enforcer = new Enforcer(); enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.SORT_PARTITION); - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -625,14 +626,14 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc ctx.setEnforcer(enforcer); ctx.setOutputPath(new Path(workDir, "grouped5")); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); assertTrue(exec instanceof SortBasedColPartitionStoreExec); } @Test public final void testPartitionedStorePlan() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), id, new FileFragment[] { frags[0] }, @@ -656,7 +657,7 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti QueryId queryId = id.getQueryUnitId().getExecutionBlockId().getQueryId(); ExecutionBlockId ebId = id.getQueryUnitId().getExecutionBlockId(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); @@ -667,7 +668,7 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti Path queryLocalTmpDir = new Path(conf.getVar(ConfVars.WORKER_TEMPORAL_DIR) + "/" + executionBlockBaseDir); FileStatus [] list = fs.listStatus(queryLocalTmpDir); - List fragments = new ArrayList(); + List fragments = new ArrayList(); for (FileStatus status : list) { assertTrue(status.isDirectory()); FileStatus [] files = fs.listStatus(status.getPath()); @@ -701,7 +702,7 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, PlanningException { // Preparing working dir and input fragments - FileFragment[] frags = StorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(), largeScore.getPath(), Integer.MAX_VALUE); QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlanWithMaxFileSize"); @@ -720,7 +721,7 @@ public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, LogicalNode rootNode = optimizer.optimize(plan); // Executing CREATE TABLE PARTITION BY - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); @@ -731,7 +732,7 @@ public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, // checking the number of partitions assertEquals(2, list.length); - List fragments = Lists.newArrayList(); + List fragments = Lists.newArrayList(); int i = 0; for (FileStatus status : list) { assertTrue(status.isDirectory()); @@ -765,7 +766,7 @@ public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, @Test public final void testPartitionedStorePlanWithEmptyGroupingSet() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan); @@ -790,7 +791,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() QueryId queryId = id.getQueryUnitId().getExecutionBlockId().getQueryId(); ExecutionBlockId ebId = id.getQueryUnitId().getExecutionBlockId(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); @@ -801,7 +802,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() Path queryLocalTmpDir = new Path(conf.getVar(ConfVars.WORKER_TEMPORAL_DIR) + "/" + executionBlockBaseDir); FileStatus [] list = fs.listStatus(queryLocalTmpDir); - List fragments = new ArrayList(); + List fragments = new ArrayList(); for (FileStatus status : list) { assertTrue(status.isDirectory()); FileStatus [] files = fs.listStatus(status.getPath()); @@ -832,7 +833,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() @Test public final void testAggregationFunction() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testAggregationFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -849,7 +850,7 @@ public final void testAggregationFunction() throws IOException, PlanningExceptio function.setFirstPhase(); } - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); @@ -863,7 +864,7 @@ public final void testAggregationFunction() throws IOException, PlanningExceptio @Test public final void testCountFunction() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCountFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -880,7 +881,7 @@ public final void testCountFunction() throws IOException, PlanningException { function.setFirstPhase(); } - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); Tuple tuple = exec.next(); @@ -891,7 +892,7 @@ public final void testCountFunction() throws IOException, PlanningException { @Test public final void testGroupByWithNullValue() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByWithNullValue"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -902,7 +903,7 @@ public final void testGroupByWithNullValue() throws IOException, PlanningExcepti LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); int count = 0; @@ -916,7 +917,7 @@ public final void testGroupByWithNullValue() throws IOException, PlanningExcepti @Test public final void testUnionPlan() throws IOException, PlanningException, CloneNotSupportedException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testUnionPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -932,7 +933,7 @@ public final void testUnionPlan() throws IOException, PlanningException, CloneNo union.setRightChild((LogicalNode) root.getChild().clone()); root.setChild(union); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, root); int count = 0; @@ -954,7 +955,7 @@ public final void testEvalExpr() throws IOException, PlanningException { LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); Tuple tuple; exec.init(); @@ -967,7 +968,7 @@ public final void testEvalExpr() throws IOException, PlanningException { plan = planner.createPlan(defaultContext, expr); rootNode = optimizer.optimize(plan); - phyPlanner = new PhysicalPlannerImpl(conf, sm); + phyPlanner = new PhysicalPlannerImpl(conf); exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); tuple = exec.next(); @@ -981,7 +982,7 @@ public final void testEvalExpr() throws IOException, PlanningException { //@Test public final void testCreateIndex() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -991,7 +992,7 @@ public final void testCreateIndex() throws IOException, PlanningException { LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); while (exec.next() != null) { @@ -1008,7 +1009,7 @@ public final void testCreateIndex() throws IOException, PlanningException { @Test public final void testDuplicateEliminate() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testDuplicateEliminate"); @@ -1020,7 +1021,7 @@ public final void testDuplicateEliminate() throws IOException, PlanningException LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); Tuple tuple; @@ -1042,7 +1043,7 @@ public final void testDuplicateEliminate() throws IOException, PlanningException @Test public final void testIndexedStoreExec() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testIndexedStoreExec"); @@ -1060,7 +1061,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()).toArray()); ctx.setDataChannel(channel); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); Tuple tuple; @@ -1135,7 +1136,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { @Test public final void testSortEnforcer() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortEnforcer"); @@ -1153,7 +1154,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); @@ -1175,7 +1176,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); - phyPlanner = new PhysicalPlannerImpl(conf,sm); + phyPlanner = new PhysicalPlannerImpl(conf); exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); @@ -1186,7 +1187,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { @Test public final void testGroupByEnforcer() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByEnforcer"); Expr context = analyzer.parse(QUERIES[7]); @@ -1203,7 +1204,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); @@ -1225,7 +1226,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); - phyPlanner = new PhysicalPlannerImpl(conf,sm); + phyPlanner = new PhysicalPlannerImpl(conf); exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); exec.next(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index c8b9e4ab04..3d1f499605 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -80,7 +80,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", TajoDataTypes.Type.INT4); @@ -89,7 +89,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.RAW); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -136,7 +136,7 @@ public void testExternalSortExecProgressWithPairWiseMerger() throws Exception { } private void testProgress(int sortBufferBytesNum) throws Exception { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -146,7 +146,7 @@ private void testProgress(int sortBufferBytesNum) throws Exception { LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = plan.getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); ProjectionExec proj = (ProjectionExec) exec; @@ -156,8 +156,7 @@ private void testProgress(int sortBufferBytesNum) throws Exception { UnaryPhysicalExec sortExec = proj.getChild(); SeqScanExec scan = sortExec.getChild(); - ExternalSortExec extSort = new ExternalSortExec(ctx, sm, - ((MemSortExec)sortExec).getPlan(), scan); + ExternalSortExec extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan); extSort.setSortBufferBytesNum(sortBufferBytesNum); proj.setChild(extSort); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 09f5ab8dec..ca1f0bba9a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -80,7 +80,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -103,7 +103,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -132,7 +132,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -171,7 +171,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -229,9 +229,9 @@ public void tearDown() throws Exception { @Test public final void testRightOuter_HashJoinExec0() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), + FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); @@ -243,7 +243,7 @@ public final void testRightOuter_HashJoinExec0() throws IOException, PlanningExc Expr expr = analyzer.parse(QUERIES[0]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -270,9 +270,9 @@ public final void testRightOuter_HashJoinExec0() throws IOException, PlanningExc @Test public final void testRightOuter_HashJoinExec1() throws IOException, PlanningException { - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -284,7 +284,7 @@ public final void testRightOuter_HashJoinExec1() throws IOException, PlanningExc Expr expr = analyzer.parse(QUERIES[1]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -311,9 +311,9 @@ public final void testRightOuter_HashJoinExec1() throws IOException, PlanningExc @Test public final void testRightOuter_HashJoinExec2() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); @@ -325,7 +325,7 @@ public final void testRightOuter_HashJoinExec2() throws IOException, PlanningExc Expr expr = analyzer.parse(QUERIES[2]); LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index f8e2f41b03..0dc8fb400e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -84,7 +84,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -107,7 +107,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -145,7 +145,7 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManager.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = StorageManager.getFileStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -176,7 +176,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -215,7 +215,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -267,7 +267,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); @@ -310,8 +310,8 @@ public final void testRightOuterMergeJoin0() throws IOException, PlanningExcepti Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); + FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin0"); @@ -319,7 +319,7 @@ public final void testRightOuterMergeJoin0() throws IOException, PlanningExcepti LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -346,9 +346,9 @@ public final void testRightOuter_MergeJoin1() throws IOException, PlanningExcept enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] job3Frags = - StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin1"); @@ -356,7 +356,7 @@ public final void testRightOuter_MergeJoin1() throws IOException, PlanningExcept LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -382,9 +382,9 @@ public final void testRightOuterMergeJoin2() throws IOException, PlanningExcepti enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] job3Frags = - StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin2"); @@ -392,7 +392,7 @@ public final void testRightOuterMergeJoin2() throws IOException, PlanningExcepti LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; assertTrue(proj.getChild() instanceof RightOuterMergeJoinExec); @@ -418,9 +418,9 @@ public final void testRightOuter_MergeJoin3() throws IOException, PlanningExcept enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] dep4Frags = - StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), dep4.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), dep4.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin3"); @@ -428,7 +428,7 @@ public final void testRightOuter_MergeJoin3() throws IOException, PlanningExcept LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; @@ -455,9 +455,9 @@ public final void testRightOuter_MergeJoin4() throws IOException, PlanningExcept enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); FileFragment[] phone3Frags = - StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -466,7 +466,7 @@ public final void testRightOuter_MergeJoin4() throws IOException, PlanningExcept LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; assertTrue(proj.getChild() instanceof RightOuterMergeJoinExec); @@ -492,8 +492,8 @@ public final void testRightOuterMergeJoin5() throws IOException, PlanningExcepti enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); @@ -503,7 +503,7 @@ public final void testRightOuterMergeJoin5() throws IOException, PlanningExcepti LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; assertTrue(proj.getChild() instanceof RightOuterMergeJoinExec); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 63e189987a..9ae683389a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -55,7 +55,7 @@ public class TestSortExec { private static SQLAnalyzer analyzer; private static LogicalPlanner planner; private static LogicalOptimizer optimizer; - private static StorageManager sm; + private static FileStorageManager sm; private static Path workDir; private static Path tablePath; private static TableMeta employeeMeta; @@ -68,7 +68,7 @@ public static void setUp() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); - sm = StorageManager.getStorageManager(conf, workDir); + sm = StorageManager.getFileStorageManager(conf, workDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -80,7 +80,7 @@ public static void setUp() throws Exception { tablePath = StorageUtil.concatPath(workDir, "employee", "table1"); sm.getFileSystem().mkdirs(tablePath.getParent()); - Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 100; i++) { @@ -108,7 +108,7 @@ public static void setUp() throws Exception { @Test public final void testNext() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility @@ -118,7 +118,7 @@ public final void testNext() throws IOException, PlanningException { LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); Tuple tuple; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index b95ae41ee4..c54494518d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -565,7 +565,7 @@ private void createMultiFile(String tableName, int numRowsEachFile, TupleCreator } Path dataPath = new Path(table.getPath(), fileIndex + ".csv"); fileIndex++; - appender = StorageManager.getStorageManager(conf).getAppender(tableMeta, schema, + appender = StorageManager.getFileStorageManager(conf).getAppender(tableMeta, schema, dataPath); appender.init(); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 4afa9c30aa..7045ba5c6e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -1026,41 +1026,42 @@ public void testComplexJoinCondition2() throws Exception { cleanupQuery(res); } - @Test - public void testComplexJoinCondition3() throws Exception { - // select n1.n_nationkey, n1.n_name, n2.n_name from nation n1 join nation n2 on lower(n1.n_name) = lower(n2.n_name); - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); - } - - @Test - public void testComplexJoinCondition4() throws Exception { - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); - } - - @Test - public void testComplexJoinCondition5() throws Exception { - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); - } - - @Test - public void testComplexJoinCondition6() throws Exception { - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); - } - - @Test - public void testComplexJoinCondition7() throws Exception { - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); - } + //FIXME The following testcase should be uncommented after resolving the issue TAJO-1126. +// @Test +// public void testComplexJoinCondition3() throws Exception { +// // select n1.n_nationkey, n1.n_name, n2.n_name from nation n1 join nation n2 on lower(n1.n_name) = lower(n2.n_name); +// ResultSet res = executeQuery(); +// assertResultSet(res); +// cleanupQuery(res); +// } +// +// @Test +// public void testComplexJoinCondition4() throws Exception { +// ResultSet res = executeQuery(); +// assertResultSet(res); +// cleanupQuery(res); +// } +// +// @Test +// public void testComplexJoinCondition5() throws Exception { +// ResultSet res = executeQuery(); +// assertResultSet(res); +// cleanupQuery(res); +// } +// +// @Test +// public void testComplexJoinCondition6() throws Exception { +// ResultSet res = executeQuery(); +// assertResultSet(res); +// cleanupQuery(res); +// } +// +// @Test +// public void testComplexJoinCondition7() throws Exception { +// ResultSet res = executeQuery(); +// assertResultSet(res); +// cleanupQuery(res); +// } @Test public void testFullOuterJoinWithEmptyIntermediateData() throws Exception { diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 1edf239d12..ab0979b408 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -55,7 +55,7 @@ public class TestResultSet { private static TajoTestingCluster util; private static TajoConf conf; private static TableDesc desc; - private static StorageManager sm; + private static FileStorageManager sm; private static TableMeta scoreMeta; private static Schema scoreSchema; @@ -63,7 +63,7 @@ public class TestResultSet { public static void setup() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); conf = util.getConfiguration(); - sm = StorageManager.getStorageManager(conf); + sm = StorageManager.getFileStorageManager(conf); scoreSchema = new Schema(); scoreSchema.addColumn("deptname", Type.TEXT); @@ -73,7 +73,7 @@ public static void setup() throws Exception { Path p = sm.getTablePath("score"); sm.getFileSystem().mkdirs(p); - Appender appender = StorageManager.getStorageManager(conf).getAppender(scoreMeta, scoreSchema, + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(scoreMeta, scoreSchema, new Path(p, "score")); appender.init(); int deptSize = 100; diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 234c58e572..98f1fa56e0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -82,7 +82,7 @@ public static void setUp() throws Exception { logicalPlanner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); - StorageManager sm = StorageManager.getStorageManager(conf); + StorageManager sm = StorageManager.getFileStorageManager(conf); dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java index e6ff7a3890..d0ab1c0d50 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java @@ -47,7 +47,7 @@ public final void testGetAndSetFields() { assertEquals("table1_1", fragment1.getTableName()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); - assertTrue(500 == fragment1.getEndKey()); + assertTrue(500 == fragment1.getLength()); } @Test @@ -58,7 +58,7 @@ public final void testGetProtoAndRestore() { assertEquals("table1_1", fragment1.getTableName()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); - assertTrue(500 == fragment1.getEndKey()); + assertTrue(500 == fragment1.getLength()); } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java index 742b07f499..2aa56db4b4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -36,6 +36,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.FileUtil; import org.junit.After; import org.junit.Before; @@ -69,7 +70,7 @@ public void test() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(StoreType.ROWFILE); - StorageManager sm = StorageManager.getStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR))); + FileStorageManager sm = StorageManager.getFileStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR))); Path tablePath = new Path("/test"); Path metaPath = new Path(tablePath, ".meta"); @@ -79,7 +80,7 @@ public void test() throws IOException { FileUtil.writeProto(fs, metaPath, meta.getProto()); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, dataPath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, dataPath); appender.enableStats(); appender.init(); @@ -110,7 +111,7 @@ public void test() throws IOException { int tupleCnt = 0; start = System.currentTimeMillis(); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); while ((tuple=scanner.next()) != null) { tupleCnt++; diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index aff5d73fad..e6d569f627 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -86,7 +86,7 @@ public void setUp() throws Exception { catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); @@ -141,7 +141,7 @@ public void testGet() throws Exception { tableDir); catalog.createTable(employee); - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE); + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), @@ -151,7 +151,7 @@ public void testGet() throws Exception { LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); ExternalSortExec sort = null; @@ -165,7 +165,7 @@ public void testGet() throws Exception { } SortSpec[] sortSpecs = sort.getPlan().getSortKeys(); - RangeShuffleFileWriteExec idxStoreExec = new RangeShuffleFileWriteExec(ctx, sm, sort, sort.getSchema(), + RangeShuffleFileWriteExec idxStoreExec = new RangeShuffleFileWriteExec(ctx, sort, sort.getSchema(), sort.getSchema(), sortSpecs); exec = idxStoreExec; @@ -264,7 +264,7 @@ public void testGetFromDescendingOrder() throws Exception { CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, tablePath); catalog.createTable(employee); - FileFragment[] frags = sm.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -275,7 +275,7 @@ public void testGetFromDescendingOrder() throws Exception { LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); + PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); ExternalSortExec sort = null; @@ -289,7 +289,7 @@ public void testGetFromDescendingOrder() throws Exception { } SortSpec[] sortSpecs = sort.getPlan().getSortKeys(); - RangeShuffleFileWriteExec idxStoreExec = new RangeShuffleFileWriteExec(ctx, sm, sort, + RangeShuffleFileWriteExec idxStoreExec = new RangeShuffleFileWriteExec(ctx, sort, sort.getSchema(), sort.getSchema(), sortSpecs); exec = idxStoreExec; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java index 211379467e..e0156581b8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java @@ -38,7 +38,7 @@ import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.storage.compress.CodecPool; import org.apache.tajo.storage.exception.AlreadyExistsStorageException; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream; import org.apache.tajo.util.BytesUtils; @@ -251,11 +251,11 @@ public String getExtension() { } public static class CSVScanner extends FileScanner implements SeekableScanner { - public CSVScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) + public CSVScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); factory = new CompressionCodecFactory(conf); - codec = factory.getCodec(fragment.getPath()); + codec = factory.getCodec(this.fragment.getPath()); if (codec == null || codec instanceof SplittableCompressionCodec) { splittable = true; } @@ -310,7 +310,7 @@ public void init() throws IOException { recordCount = 0; pos = startOffset = fragment.getStartKey(); - end = startOffset + fragment.getEndKey(); + end = startOffset + fragment.getLength(); if (codec != null) { decompressor = CodecPool.getDecompressor(codec); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileScanner.java index f15c4c97a5..038f0f4c78 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileScanner.java @@ -30,6 +30,7 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; @@ -49,11 +50,11 @@ public abstract class FileScanner implements Scanner { protected TableStats tableStats; - public FileScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) { + public FileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) { this.conf = conf; this.meta = meta; this.schema = schema; - this.fragment = fragment; + this.fragment = (FileFragment)fragment; this.tableStats = new TableStats(); this.columnNum = this.schema.size(); } @@ -63,7 +64,7 @@ public void init() throws IOException { progress = 0.0f; if (fragment != null) { - tableStats.setNumBytes(fragment.getEndKey()); + tableStats.setNumBytes(fragment.getLength()); tableStats.setNumBlocks(1); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java new file mode 100644 index 0000000000..c925cd8b79 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -0,0 +1,702 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.fragment.FragmentConvertor; +import org.apache.tajo.util.*; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.net.URI; +import java.util.*; + +public class FileStorageManager extends StorageManager { + private final Log LOG = LogFactory.getLog(FileStorageManager.class); + + protected FileSystem fs; + protected Path tableBaseDir; + protected boolean blocksMetadataEnabled; + private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); + + @Override + protected void storageInit() throws IOException { + this.tableBaseDir = TajoConf.getWarehouseDir(conf); + this.fs = tableBaseDir.getFileSystem(conf); + this.blocksMetadataEnabled = conf.getBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, + DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT); + if (!this.blocksMetadataEnabled) + LOG.warn("does not support block metadata. ('dfs.datanode.hdfs-blocks-metadata.enabled')"); + } + + public Scanner getFileScanner(TableMeta meta, Schema schema, Path path) + throws IOException { + FileSystem fs = path.getFileSystem(conf); + FileStatus status = fs.getFileStatus(path); + return getFileScanner(meta, schema, path, status); + } + + public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) + throws IOException { + Fragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); + return getScanner(meta, schema, fragment); + } + + public FileSystem getFileSystem() { + return this.fs; + } + + public Path getWarehouseDir() { + return this.tableBaseDir; + } + + public void delete(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + fs.delete(tablePath, true); + } + + public boolean exists(Path path) throws IOException { + FileSystem fileSystem = path.getFileSystem(conf); + return fileSystem.exists(path); + } + + /** + * This method deletes only data contained in the given path. + * + * @param path The path in which data are deleted. + * @throws IOException + */ + public void deleteData(Path path) throws IOException { + FileSystem fileSystem = path.getFileSystem(conf); + FileStatus[] fileLists = fileSystem.listStatus(path); + for (FileStatus status : fileLists) { + fileSystem.delete(status.getPath(), true); + } + } + + public Path getTablePath(String tableName) { + return new Path(tableBaseDir, tableName); + } + + public Appender getAppender(TableMeta meta, Schema schema, Path path) + throws IOException { + Appender appender; + + Class appenderClass; + + String handlerName = meta.getStoreType().name().toLowerCase(); + appenderClass = APPENDER_HANDLER_CACHE.get(handlerName); + if (appenderClass == null) { + appenderClass = conf.getClass( + String.format("tajo.storage.appender-handler.%s.class", + meta.getStoreType().name().toLowerCase()), null, + FileAppender.class); + APPENDER_HANDLER_CACHE.put(handlerName, appenderClass); + } + + if (appenderClass == null) { + throw new IOException("Unknown Storage Type: " + meta.getStoreType()); + } + + appender = newAppenderInstance(appenderClass, conf, meta, schema, path); + + return appender; + } + + public TableMeta getTableMeta(Path tablePath) throws IOException { + TableMeta meta; + + FileSystem fs = tablePath.getFileSystem(conf); + Path tableMetaPath = new Path(tablePath, ".meta"); + if (!fs.exists(tableMetaPath)) { + throw new FileNotFoundException(".meta file not found in " + tablePath.toString()); + } + + FSDataInputStream tableMetaIn = fs.open(tableMetaPath); + + CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) org.apache.tajo.util.FileUtil.loadProto(tableMetaIn, + CatalogProtos.TableProto.getDefaultInstance()); + meta = new TableMeta(tableProto); + + return meta; + } + + public FileFragment[] split(String tableName) throws IOException { + Path tablePath = new Path(tableBaseDir, tableName); + return split(tableName, tablePath, fs.getDefaultBlockSize()); + } + + public FileFragment[] split(String tableName, long fragmentSize) throws IOException { + Path tablePath = new Path(tableBaseDir, tableName); + return split(tableName, tablePath, fragmentSize); + } + + public FileFragment[] splitBroadcastTable(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + List listTablets = new ArrayList(); + FileFragment tablet; + + FileStatus[] fileLists = fs.listStatus(tablePath); + for (FileStatus file : fileLists) { + tablet = new FileFragment(tablePath.getName(), file.getPath(), 0, file.getLen()); + listTablets.add(tablet); + } + + FileFragment[] tablets = new FileFragment[listTablets.size()]; + listTablets.toArray(tablets); + + return tablets; + } + + public FileFragment[] split(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + return split(tablePath.getName(), tablePath, fs.getDefaultBlockSize()); + } + + public FileFragment[] split(String tableName, Path tablePath) throws IOException { + return split(tableName, tablePath, fs.getDefaultBlockSize()); + } + + private FileFragment[] split(String tableName, Path tablePath, long size) + throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + + long defaultBlockSize = size; + List listTablets = new ArrayList(); + FileFragment tablet; + + FileStatus[] fileLists = fs.listStatus(tablePath); + for (FileStatus file : fileLists) { + long remainFileSize = file.getLen(); + long start = 0; + if (remainFileSize > defaultBlockSize) { + while (remainFileSize > defaultBlockSize) { + tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); + listTablets.add(tablet); + start += defaultBlockSize; + remainFileSize -= defaultBlockSize; + } + listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); + } else { + listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); + } + } + + FileFragment[] tablets = new FileFragment[listTablets.size()]; + listTablets.toArray(tablets); + + return tablets; + } + + public static FileFragment[] splitNG(Configuration conf, String tableName, TableMeta meta, + Path tablePath, long size) + throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + + long defaultBlockSize = size; + List listTablets = new ArrayList(); + FileFragment tablet; + + FileStatus[] fileLists = fs.listStatus(tablePath); + for (FileStatus file : fileLists) { + long remainFileSize = file.getLen(); + long start = 0; + if (remainFileSize > defaultBlockSize) { + while (remainFileSize > defaultBlockSize) { + tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); + listTablets.add(tablet); + start += defaultBlockSize; + remainFileSize -= defaultBlockSize; + } + listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); + } else { + listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); + } + } + + FileFragment[] tablets = new FileFragment[listTablets.size()]; + listTablets.toArray(tablets); + + return tablets; + } + + public long calculateSize(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + long totalSize = 0; + + if (fs.exists(tablePath)) { + totalSize = fs.getContentSummary(tablePath).getLength(); + } + + return totalSize; + } + + ///////////////////////////////////////////////////////////////////////////// + // FileInputFormat Area + ///////////////////////////////////////////////////////////////////////////// + + public static final PathFilter hiddenFileFilter = new PathFilter() { + public boolean accept(Path p) { + String name = p.getName(); + return !name.startsWith("_") && !name.startsWith("."); + } + }; + + /** + * Proxy PathFilter that accepts a path only if all filters given in the + * constructor do. Used by the listPaths() to apply the built-in + * hiddenFileFilter together with a user provided one (if any). + */ + private static class MultiPathFilter implements PathFilter { + private List filters; + + public MultiPathFilter(List filters) { + this.filters = filters; + } + + public boolean accept(Path path) { + for (PathFilter filter : filters) { + if (!filter.accept(path)) { + return false; + } + } + return true; + } + } + + /** + * List input directories. + * Subclasses may override to, e.g., select only files matching a regular + * expression. + * + * @return array of FileStatus objects + * @throws IOException if zero items. + */ + protected List listStatus(Path... dirs) throws IOException { + List result = new ArrayList(); + if (dirs.length == 0) { + throw new IOException("No input paths specified in job"); + } + + List errors = new ArrayList(); + + // creates a MultiPathFilter with the hiddenFileFilter and the + // user provided one (if any). + List filters = new ArrayList(); + filters.add(hiddenFileFilter); + + PathFilter inputFilter = new MultiPathFilter(filters); + + for (int i = 0; i < dirs.length; ++i) { + Path p = dirs[i]; + + FileSystem fs = p.getFileSystem(conf); + FileStatus[] matches = fs.globStatus(p, inputFilter); + if (matches == null) { + errors.add(new IOException("Input path does not exist: " + p)); + } else if (matches.length == 0) { + errors.add(new IOException("Input Pattern " + p + " matches 0 files")); + } else { + for (FileStatus globStat : matches) { + if (globStat.isDirectory()) { + for (FileStatus stat : fs.listStatus(globStat.getPath(), + inputFilter)) { + result.add(stat); + } + } else { + result.add(globStat); + } + } + } + } + + if (!errors.isEmpty()) { + throw new InvalidInputException(errors); + } + LOG.info("Total input paths to process : " + result.size()); + return result; + } + + /** + * Is the given filename splitable? Usually, true, but if the file is + * stream compressed, it will not be. + *

+ * FileInputFormat implementations can override this and return + * false to ensure that individual input files are never split-up + * so that Mappers process entire files. + * + * + * @param path the file name to check + * @param status get the file length + * @return is this file isSplittable? + */ + protected boolean isSplittable(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { + Scanner scanner = getFileScanner(meta, schema, path, status); + boolean split = scanner.isSplittable(); + scanner.close(); + return split; + } + + private static final double SPLIT_SLOP = 1.1; // 10% slop + + protected int getBlockIndex(BlockLocation[] blkLocations, + long offset) { + for (int i = 0; i < blkLocations.length; i++) { + // is the offset inside this block? + if ((blkLocations[i].getOffset() <= offset) && + (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())) { + return i; + } + } + BlockLocation last = blkLocations[blkLocations.length - 1]; + long fileLength = last.getOffset() + last.getLength() - 1; + throw new IllegalArgumentException("Offset " + offset + + " is outside of file (0.." + + fileLength + ")"); + } + + /** + * A factory that makes the split for this class. It can be overridden + * by sub-classes to make sub-types + */ + protected FileFragment makeSplit(String fragmentId, Path file, long start, long length) { + return new FileFragment(fragmentId, file, start, length); + } + + protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, + String[] hosts) { + return new FileFragment(fragmentId, file, start, length, hosts); + } + + protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blockLocation) + throws IOException { + return new FileFragment(fragmentId, file, blockLocation); + } + + // for Non Splittable. eg, compressed gzip TextFile + protected FileFragment makeNonSplit(String fragmentId, Path file, long start, long length, + BlockLocation[] blkLocations) throws IOException { + + Map hostsBlockMap = new HashMap(); + for (BlockLocation blockLocation : blkLocations) { + for (String host : blockLocation.getHosts()) { + if (hostsBlockMap.containsKey(host)) { + hostsBlockMap.put(host, hostsBlockMap.get(host) + 1); + } else { + hostsBlockMap.put(host, 1); + } + } + } + + List> entries = new ArrayList>(hostsBlockMap.entrySet()); + Collections.sort(entries, new Comparator>() { + + @Override + public int compare(Map.Entry v1, Map.Entry v2) { + return v1.getValue().compareTo(v2.getValue()); + } + }); + + String[] hosts = new String[blkLocations[0].getHosts().length]; + + for (int i = 0; i < hosts.length; i++) { + Map.Entry entry = entries.get((entries.size() - 1) - i); + hosts[i] = entry.getKey(); + } + return new FileFragment(fragmentId, file, start, length, hosts); + } + + /** + * Get the minimum split size + * + * @return the minimum number of bytes that can be in a split + */ + public long getMinSplitSize() { + return conf.getLongVar(TajoConf.ConfVars.MINIMUM_SPLIT_SIZE); + } + + /** + * Get Disk Ids by Volume Bytes + */ + private int[] getDiskIds(VolumeId[] volumeIds) { + int[] diskIds = new int[volumeIds.length]; + for (int i = 0; i < volumeIds.length; i++) { + int diskId = -1; + if (volumeIds[i] != null && volumeIds[i].hashCode() > 0) { + diskId = volumeIds[i].hashCode() - zeroVolumeId.hashCode(); + } + diskIds[i] = diskId; + } + return diskIds; + } + + /** + * Generate the map of host and make them into Volume Ids. + * + */ + private Map> getVolumeMap(List frags) { + Map> volumeMap = new HashMap>(); + for (FileFragment frag : frags) { + String[] hosts = frag.getHosts(); + int[] diskIds = frag.getDiskIds(); + for (int i = 0; i < hosts.length; i++) { + Set volumeList = volumeMap.get(hosts[i]); + if (volumeList == null) { + volumeList = new HashSet(); + volumeMap.put(hosts[i], volumeList); + } + + if (diskIds.length > 0 && diskIds[i] > -1) { + volumeList.add(diskIds[i]); + } + } + } + + return volumeMap; + } + /** + * Generate the list of files and make them into FileSplits. + * + * @throws IOException + */ + public List getSplits(String tableName, TableMeta meta, Schema schema, Path... inputs) + throws IOException { + // generate splits' + + List splits = Lists.newArrayList(); + List volumeSplits = Lists.newArrayList(); + List blockLocations = Lists.newArrayList(); + + for (Path p : inputs) { + FileSystem fs = p.getFileSystem(conf); + ArrayList files = Lists.newArrayList(); + if (fs.isFile(p)) { + files.addAll(Lists.newArrayList(fs.getFileStatus(p))); + } else { + files.addAll(listStatus(p)); + } + + int previousSplitSize = splits.size(); + for (FileStatus file : files) { + Path path = file.getPath(); + long length = file.getLen(); + if (length > 0) { + // Get locations of blocks of file + BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); + boolean splittable = isSplittable(meta, schema, path, file); + if (blocksMetadataEnabled && fs instanceof DistributedFileSystem) { + + if (splittable) { + for (BlockLocation blockLocation : blkLocations) { + volumeSplits.add(makeSplit(tableName, path, blockLocation)); + } + blockLocations.addAll(Arrays.asList(blkLocations)); + + } else { // Non splittable + long blockSize = blkLocations[0].getLength(); + if (blockSize >= length) { + blockLocations.addAll(Arrays.asList(blkLocations)); + for (BlockLocation blockLocation : blkLocations) { + volumeSplits.add(makeSplit(tableName, path, blockLocation)); + } + } else { + splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); + } + } + + } else { + if (splittable) { + + long minSize = Math.max(getMinSplitSize(), 1); + + long blockSize = file.getBlockSize(); // s3n rest api contained block size but blockLocations is one + long splitSize = Math.max(minSize, blockSize); + long bytesRemaining = length; + + // for s3 + while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) { + int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); + splits.add(makeSplit(tableName, path, length - bytesRemaining, splitSize, + blkLocations[blkIndex].getHosts())); + bytesRemaining -= splitSize; + } + if (bytesRemaining > 0) { + int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); + splits.add(makeSplit(tableName, path, length - bytesRemaining, bytesRemaining, + blkLocations[blkIndex].getHosts())); + } + } else { // Non splittable + splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); + } + } + } else { + //for zero length files + splits.add(makeSplit(tableName, path, 0, length)); + } + } + if(LOG.isDebugEnabled()){ + LOG.debug("# of splits per partition: " + (splits.size() - previousSplitSize)); + } + } + + // Combine original fileFragments with new VolumeId information + setVolumeMeta(volumeSplits, blockLocations); + splits.addAll(volumeSplits); + LOG.info("Total # of splits: " + splits.size()); + return splits; + } + + private void setVolumeMeta(List splits, final List blockLocations) + throws IOException { + + int locationSize = blockLocations.size(); + int splitSize = splits.size(); + if (locationSize == 0 || splitSize == 0) return; + + if (locationSize != splitSize) { + // splits and locations don't match up + LOG.warn("Number of block locations not equal to number of splits: " + + "#locations=" + locationSize + + " #splits=" + splitSize); + return; + } + + DistributedFileSystem fs = (DistributedFileSystem)DistributedFileSystem.get(conf); + int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); + int blockLocationIdx = 0; + + Iterator iter = splits.iterator(); + while (locationSize > blockLocationIdx) { + + int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); + List locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); + //BlockStorageLocation containing additional volume location information for each replica of each block. + BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); + + for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { + ((FileFragment)iter.next()).setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); + blockLocationIdx++; + } + } + LOG.info("# of splits with volumeId " + splitSize); + } + + private static class InvalidInputException extends IOException { + List errors; + public InvalidInputException(List errors) { + this.errors = errors; + } + + @Override + public String getMessage(){ + StringBuffer sb = new StringBuffer(); + int messageLimit = Math.min(errors.size(), 10); + for (int i = 0; i < messageLimit ; i ++) { + sb.append(errors.get(i).getMessage()).append("\n"); + } + + if(messageLimit < errors.size()) + sb.append("skipped .....").append("\n"); + + return sb.toString(); + } + } + + @Override + public List getSplits(String tableName, TableDesc table) throws IOException { + return getSplits(tableName, table.getMeta(), table.getSchema(), table.getPath()); + } + + @Override + public void createTable(TableDesc tableDesc) throws IOException { + if (!tableDesc.isExternal()) { + String [] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); + String databaseName = splitted[0]; + String simpleTableName = splitted[1]; + + // create a table directory (i.e., ${WAREHOUSE_DIR}/${DATABASE_NAME}/${TABLE_NAME} ) + Path tablePath = StorageUtil.concatPath(tableBaseDir, databaseName, simpleTableName); + tableDesc.setPath(tablePath); + } else { + Preconditions.checkState(tableDesc.getPath() != null, "ERROR: LOCATION must be given."); + } + + Path path = tableDesc.getPath(); + + FileSystem fs = path.getFileSystem(conf); + TableStats stats = new TableStats(); + if (tableDesc.isExternal()) { + if (!fs.exists(path)) { + LOG.error(path.toUri() + " does not exist"); + throw new IOException("ERROR: " + path.toUri() + " does not exist"); + } + } else { + fs.mkdirs(path); + } + + long totalSize = 0; + + try { + totalSize = calculateSize(path); + } catch (IOException e) { + LOG.warn("Cannot calculate the size of the relation", e); + } + + stats.setNumBytes(totalSize); + + if (tableDesc.isExternal()) { // if it is an external table, there is no way to know the exact row number without processing. + stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); + } + + tableDesc.setStats(stats); + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException { + try { + Path path = tableDesc.getPath(); + FileSystem fs = path.getFileSystem(conf); + LOG.info("Delete table data dir: " + path); + fs.delete(path, true); + } catch (IOException e) { + throw new InternalError(e.getMessage()); + } + } + +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index 84d81d5659..ece31dd9b5 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -85,8 +85,8 @@ public HashShuffleAppender getAppender(TajoConf tajoConf, ExecutionBlockId ebId, if (!fs.exists(dataFile.getParent())) { fs.mkdirs(dataFile.getParent()); } - FileAppender appender = (FileAppender) StorageManager.getStorageManager( - tajoConf).getAppender(meta, outSchema, dataFile); + FileAppender appender = (FileAppender) StorageManager.getFileStorageManager( + tajoConf, null).getAppender(meta, outSchema, dataFile); appender.enableStats(); appender.init(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java index 4122c76d66..89e59d08b7 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -18,7 +18,6 @@ package org.apache.tajo.storage; -import com.google.common.collect.ImmutableList; import org.apache.hadoop.conf.Configuration; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; @@ -26,7 +25,7 @@ import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; import java.util.ArrayList; @@ -37,9 +36,9 @@ public class MergeScanner implements Scanner { private Configuration conf; private TableMeta meta; private Schema schema; - private List fragments; - private Iterator iterator; - private FileFragment currentFragment; + private List fragments; + private Iterator iterator; + private Fragment currentFragment; private Scanner currentScanner; private Tuple tuple; private boolean projectable = false; @@ -48,12 +47,12 @@ public class MergeScanner implements Scanner { private float progress; protected TableStats tableStats; - public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List rawFragmentList) + public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List rawFragmentList) throws IOException { this(conf, schema, meta, rawFragmentList, schema); } - public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List rawFragmentList, + public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List rawFragmentList, Schema target) throws IOException { this.conf = conf; @@ -61,12 +60,12 @@ public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List(); + this.fragments = new ArrayList(); long numBytes = 0; - for (FileFragment eachFileFragment: rawFragmentList) { - numBytes += eachFileFragment.getEndKey(); - if (eachFileFragment.getEndKey() > 0) { + for (Fragment eachFileFragment: rawFragmentList) { + if (eachFileFragment.getLength() > 0) { + numBytes += eachFileFragment.getLength(); fragments.add(eachFileFragment); } } @@ -128,7 +127,7 @@ public void reset() throws IOException { private Scanner getNextScanner() throws IOException { if (iterator.hasNext()) { currentFragment = iterator.next(); - currentScanner = StorageManager.getStorageManager((TajoConf)conf).getScanner(meta, schema, + currentScanner = StorageManager.getStorageManager((TajoConf)conf, meta.getStoreType()).getScanner(meta, schema, currentFragment, target); currentScanner.init(); return currentScanner; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/NullScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/NullScanner.java index 4cec67d39d..19d333e066 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/NullScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/NullScanner.java @@ -19,12 +19,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; public class NullScanner extends FileScanner { - public NullScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) { + public NullScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { super(conf, schema, meta, fragment); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java index c8ac3a2141..f72a5a13c8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java @@ -32,7 +32,7 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.ProtobufDatumFactory; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.BitArray; @@ -63,7 +63,7 @@ public static class RawFileScanner extends FileScanner implements SeekableScanne private FileInputStream fis; private long recordCount; - public RawFileScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException { + public RawFileScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { super(conf, schema, meta, fragment); } @@ -81,18 +81,18 @@ public void init() throws IOException { fis = new FileInputStream(file); channel = fis.getChannel(); - fileLimit = fragment.getStartKey() + fragment.getEndKey(); // fileLimit is less than or equal to fileSize + fileLimit = fragment.getStartKey() + fragment.getLength(); // fileLimit is less than or equal to fileSize if (tableStats != null) { - tableStats.setNumBytes(fragment.getEndKey()); + tableStats.setNumBytes(fragment.getLength()); } if (LOG.isDebugEnabled()) { LOG.debug("RawFileScanner open:" + fragment + "," + channel.position() + ", total file size :" + channel.size() - + ", fragment size :" + fragment.getEndKey() + ", fileLimit: " + fileLimit); + + ", fragment size :" + fragment.getLength() + ", fileLimit: " + fileLimit); } - if (fragment.getEndKey() < 64 * StorageUnit.KB) { - bufferSize = fragment.getEndKey().intValue(); + if (fragment.getLength() < 64 * StorageUnit.KB) { + bufferSize = (int)fragment.getLength(); } else { bufferSize = 64 * StorageUnit.KB; } @@ -138,7 +138,7 @@ public void seek(long offset) throws IOException { } private boolean fillBuffer() throws IOException { - if (numBytesRead >= fragment.getEndKey()) { + if (numBytesRead >= fragment.getLength()) { eof = true; return false; } @@ -150,7 +150,7 @@ private boolean fillBuffer() throws IOException { return false; } else { buffer.flip(); - long realRemaining = fragment.getEndKey() - numBytesRead; + long realRemaining = fragment.getLength() - numBytesRead; numBytesRead += bytesRead; if (realRemaining < bufferSize) { int newLimit = currentDataSize + (int) realRemaining; @@ -397,7 +397,7 @@ public void reset() throws IOException { @Override public void close() throws IOException { if (tableStats != null) { - tableStats.setReadBytes(fragment.getEndKey()); + tableStats.setReadBytes(fragment.getLength()); tableStats.setNumRows(recordCount); } @@ -431,14 +431,14 @@ public float getProgress() { } if(eof || channel == null) { - tableStats.setReadBytes(fragment.getEndKey()); + tableStats.setReadBytes(fragment.getLength()); return 1.0f; } if (filePos == 0) { return 0.0f; } else { - return Math.min(1.0f, ((float)filePos / fragment.getEndKey().floatValue())); + return Math.min(1.0f, ((float)filePos / (float)fragment.getLength())); } } catch (IOException e) { LOG.error(e.getMessage(), e); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java index db36771835..640cae20c0 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java @@ -33,7 +33,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.exception.AlreadyExistsStorageException; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.BitArray; import java.io.FileNotFoundException; @@ -66,7 +66,7 @@ public static class RowFileScanner extends FileScanner { private BitArray nullFlags; private long bufferStartPos; - public RowFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) + public RowFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); @@ -75,8 +75,8 @@ public RowFileScanner(Configuration conf, final Schema schema, final TableMeta m nullFlags = new BitArray(schema.size()); tupleHeaderSize = nullFlags.bytesLength() + (2 * Short.SIZE / 8); - this.start = fragment.getStartKey(); - this.end = this.start + fragment.getEndKey(); + this.start = this.fragment.getStartKey(); + this.end = this.start + this.fragment.getLength(); } public void init() throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 87b41973a1..23c2406804 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -27,9 +27,11 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.storage.fragment.FileFragment; @@ -48,14 +50,10 @@ /** * StorageManager */ -public class StorageManager { +public abstract class StorageManager { private final Log LOG = LogFactory.getLog(StorageManager.class); - protected final TajoConf conf; - protected final FileSystem fs; - protected final Path tableBaseDir; - protected final boolean blocksMetadataEnabled; - private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); + protected TajoConf conf; private static final Map storageManagers = Maps.newHashMap(); @@ -78,58 +76,52 @@ public class StorageManager { private static final Map, Constructor> CONSTRUCTOR_CACHE = new ConcurrentHashMap, Constructor>(); - private StorageManager(TajoConf conf) throws IOException { - this.conf = conf; - this.tableBaseDir = TajoConf.getWarehouseDir(conf); - this.fs = tableBaseDir.getFileSystem(conf); - this.blocksMetadataEnabled = conf.getBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, - DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT); - if (!this.blocksMetadataEnabled) - LOG.warn("does not support block metadata. ('dfs.datanode.hdfs-blocks-metadata.enabled')"); - } + protected abstract void storageInit() throws IOException ; + public abstract void createTable(TableDesc tableDesc) throws IOException; + public abstract void purgeTable(TableDesc tableDesc) throws IOException; + public abstract List getSplits(String fragmentId, TableDesc tableDesc) throws IOException; - public static StorageManager getStorageManager(TajoConf conf) throws IOException { - return getStorageManager(conf, null); + public void init(TajoConf tajoConf) throws IOException { + this.conf = tajoConf; + storageInit(); } - public static synchronized StorageManager getStorageManager ( - TajoConf conf, Path warehouseDir) throws IOException { + public static FileStorageManager getFileStorageManager(TajoConf tajoConf) throws IOException { + return getFileStorageManager(tajoConf, null); + } + public static FileStorageManager getFileStorageManager(TajoConf tajoConf, Path warehousePath) throws IOException { URI uri; - TajoConf localConf = new TajoConf(conf); - if (warehouseDir != null) { - localConf.setVar(ConfVars.WAREHOUSE_DIR, warehouseDir.toUri().toString()); + TajoConf copiedConf = new TajoConf(tajoConf); + if (warehousePath != null) { + copiedConf.setVar(ConfVars.WAREHOUSE_DIR, warehousePath.toUri().toString()); } - - uri = TajoConf.getWarehouseDir(localConf).toUri(); - + uri = TajoConf.getWarehouseDir(copiedConf).toUri(); String key = "file".equals(uri.getScheme()) ? "file" : uri.toString(); - - if(storageManagers.containsKey(key)) { - StorageManager sm = storageManagers.get(key); - return sm; - } else { - StorageManager storageManager = new StorageManager(localConf); - storageManagers.put(key, storageManager); - return storageManager; - } + return (FileStorageManager) getStorageManager(copiedConf, StoreType.CSV, key); } - public Scanner getFileScanner(TableMeta meta, Schema schema, Path path) - throws IOException { - FileSystem fs = path.getFileSystem(conf); - FileStatus status = fs.getFileStatus(path); - return getFileScanner(meta, schema, path, status); + public static StorageManager getStorageManager(TajoConf tajoConf, StoreType storeType) throws IOException { + return getStorageManager(tajoConf, storeType, null); } - public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) - throws IOException { - FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); - return getScanner(meta, schema, fragment); - } + public static synchronized StorageManager getStorageManager ( + TajoConf conf, StoreType storeType, String managerKey) throws IOException { + synchronized (storageManagers) { + String storeKey = storeType + managerKey; + StorageManager manager = storageManagers.get(storeKey); + if (manager == null) { + switch (storeType) { + default: + manager = new FileStorageManager(); + } - public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment) throws IOException { - return getScanner(meta, schema, FragmentConvertor.convert(conf, meta.getStoreType(), fragment), schema); + manager.init(conf); + storageManagers.put(storeKey, manager); + } + + return manager; + } } public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException { @@ -140,42 +132,6 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) thro return getScanner(meta, schema, fragment, schema); } - public FileSystem getFileSystem() { - return this.fs; - } - - public Path getWarehouseDir() { - return this.tableBaseDir; - } - - public void delete(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - fs.delete(tablePath, true); - } - - public boolean exists(Path path) throws IOException { - FileSystem fileSystem = path.getFileSystem(conf); - return fileSystem.exists(path); - } - - /** - * This method deletes only data contained in the given path. - * - * @param path The path in which data are deleted. - * @throws IOException - */ - public void deleteData(Path path) throws IOException { - FileSystem fileSystem = path.getFileSystem(conf); - FileStatus[] fileLists = fileSystem.listStatus(path); - for (FileStatus status : fileLists) { - fileSystem.delete(status.getPath(), true); - } - } - - public Path getTablePath(String tableName) { - return new Path(tableBaseDir, tableName); - } - public Appender getAppender(TableMeta meta, Schema schema, Path path) throws IOException { Appender appender; @@ -201,510 +157,11 @@ public Appender getAppender(TableMeta meta, Schema schema, Path path) return appender; } - public TableMeta getTableMeta(Path tablePath) throws IOException { - TableMeta meta; - - FileSystem fs = tablePath.getFileSystem(conf); - Path tableMetaPath = new Path(tablePath, ".meta"); - if (!fs.exists(tableMetaPath)) { - throw new FileNotFoundException(".meta file not found in " + tablePath.toString()); - } - - FSDataInputStream tableMetaIn = fs.open(tableMetaPath); - - CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) FileUtil.loadProto(tableMetaIn, - CatalogProtos.TableProto.getDefaultInstance()); - meta = new TableMeta(tableProto); - - return meta; - } - - public FileFragment[] split(String tableName) throws IOException { - Path tablePath = new Path(tableBaseDir, tableName); - return split(tableName, tablePath, fs.getDefaultBlockSize()); - } - - public FileFragment[] split(String tableName, long fragmentSize) throws IOException { - Path tablePath = new Path(tableBaseDir, tableName); - return split(tableName, tablePath, fragmentSize); - } - - public FileFragment[] splitBroadcastTable(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - List listTablets = new ArrayList(); - FileFragment tablet; - - FileStatus[] fileLists = fs.listStatus(tablePath); - for (FileStatus file : fileLists) { - tablet = new FileFragment(tablePath.getName(), file.getPath(), 0, file.getLen()); - listTablets.add(tablet); - } - - FileFragment[] tablets = new FileFragment[listTablets.size()]; - listTablets.toArray(tablets); - - return tablets; - } - - public FileFragment[] split(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - return split(tablePath.getName(), tablePath, fs.getDefaultBlockSize()); - } - - public FileFragment[] split(String tableName, Path tablePath) throws IOException { - return split(tableName, tablePath, fs.getDefaultBlockSize()); - } - - private FileFragment[] split(String tableName, Path tablePath, long size) - throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - - long defaultBlockSize = size; - List listTablets = new ArrayList(); - FileFragment tablet; - - FileStatus[] fileLists = fs.listStatus(tablePath); - for (FileStatus file : fileLists) { - long remainFileSize = file.getLen(); - long start = 0; - if (remainFileSize > defaultBlockSize) { - while (remainFileSize > defaultBlockSize) { - tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); - listTablets.add(tablet); - start += defaultBlockSize; - remainFileSize -= defaultBlockSize; - } - listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); - } else { - listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); - } - } - - FileFragment[] tablets = new FileFragment[listTablets.size()]; - listTablets.toArray(tablets); - - return tablets; - } - - public static FileFragment[] splitNG(Configuration conf, String tableName, TableMeta meta, - Path tablePath, long size) - throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - - long defaultBlockSize = size; - List listTablets = new ArrayList(); - FileFragment tablet; - - FileStatus[] fileLists = fs.listStatus(tablePath); - for (FileStatus file : fileLists) { - long remainFileSize = file.getLen(); - long start = 0; - if (remainFileSize > defaultBlockSize) { - while (remainFileSize > defaultBlockSize) { - tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); - listTablets.add(tablet); - start += defaultBlockSize; - remainFileSize -= defaultBlockSize; - } - listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); - } else { - listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); - } - } - - FileFragment[] tablets = new FileFragment[listTablets.size()]; - listTablets.toArray(tablets); - - return tablets; - } - - public long calculateSize(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - long totalSize = 0; - - if (fs.exists(tablePath)) { - totalSize = fs.getContentSummary(tablePath).getLength(); - } - - return totalSize; - } - - ///////////////////////////////////////////////////////////////////////////// - // FileInputFormat Area - ///////////////////////////////////////////////////////////////////////////// - - public static final PathFilter hiddenFileFilter = new PathFilter() { - public boolean accept(Path p) { - String name = p.getName(); - return !name.startsWith("_") && !name.startsWith("."); - } - }; - - /** - * Proxy PathFilter that accepts a path only if all filters given in the - * constructor do. Used by the listPaths() to apply the built-in - * hiddenFileFilter together with a user provided one (if any). - */ - private static class MultiPathFilter implements PathFilter { - private List filters; - - public MultiPathFilter(List filters) { - this.filters = filters; - } - - public boolean accept(Path path) { - for (PathFilter filter : filters) { - if (!filter.accept(path)) { - return false; - } - } - return true; - } - } - - /** - * List input directories. - * Subclasses may override to, e.g., select only files matching a regular - * expression. - * - * @return array of FileStatus objects - * @throws IOException if zero items. - */ - protected List listStatus(Path... dirs) throws IOException { - List result = new ArrayList(); - if (dirs.length == 0) { - throw new IOException("No input paths specified in job"); - } - - List errors = new ArrayList(); - - // creates a MultiPathFilter with the hiddenFileFilter and the - // user provided one (if any). - List filters = new ArrayList(); - filters.add(hiddenFileFilter); - - PathFilter inputFilter = new MultiPathFilter(filters); - - for (int i = 0; i < dirs.length; ++i) { - Path p = dirs[i]; - - FileSystem fs = p.getFileSystem(conf); - FileStatus[] matches = fs.globStatus(p, inputFilter); - if (matches == null) { - errors.add(new IOException("Input path does not exist: " + p)); - } else if (matches.length == 0) { - errors.add(new IOException("Input Pattern " + p + " matches 0 files")); - } else { - for (FileStatus globStat : matches) { - if (globStat.isDirectory()) { - for (FileStatus stat : fs.listStatus(globStat.getPath(), - inputFilter)) { - result.add(stat); - } - } else { - result.add(globStat); - } - } - } - } - - if (!errors.isEmpty()) { - throw new InvalidInputException(errors); - } - LOG.info("Total input paths to process : " + result.size()); - return result; - } - - /** - * Is the given filename splitable? Usually, true, but if the file is - * stream compressed, it will not be. - *

- * FileInputFormat implementations can override this and return - * false to ensure that individual input files are never split-up - * so that Mappers process entire files. - * - * - * @param path the file name to check - * @param status get the file length - * @return is this file isSplittable? - */ - protected boolean isSplittable(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { - Scanner scanner = getFileScanner(meta, schema, path, status); - boolean split = scanner.isSplittable(); - scanner.close(); - return split; - } - - private static final double SPLIT_SLOP = 1.1; // 10% slop - - protected int getBlockIndex(BlockLocation[] blkLocations, - long offset) { - for (int i = 0; i < blkLocations.length; i++) { - // is the offset inside this block? - if ((blkLocations[i].getOffset() <= offset) && - (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())) { - return i; - } - } - BlockLocation last = blkLocations[blkLocations.length - 1]; - long fileLength = last.getOffset() + last.getLength() - 1; - throw new IllegalArgumentException("Offset " + offset + - " is outside of file (0.." + - fileLength + ")"); - } - - /** - * A factory that makes the split for this class. It can be overridden - * by sub-classes to make sub-types - */ - protected FileFragment makeSplit(String fragmentId, Path file, long start, long length) { - return new FileFragment(fragmentId, file, start, length); - } - - protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, - String[] hosts) { - return new FileFragment(fragmentId, file, start, length, hosts); - } - - protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blockLocation) - throws IOException { - return new FileFragment(fragmentId, file, blockLocation); - } - - // for Non Splittable. eg, compressed gzip TextFile - protected FileFragment makeNonSplit(String fragmentId, Path file, long start, long length, - BlockLocation[] blkLocations) throws IOException { - - Map hostsBlockMap = new HashMap(); - for (BlockLocation blockLocation : blkLocations) { - for (String host : blockLocation.getHosts()) { - if (hostsBlockMap.containsKey(host)) { - hostsBlockMap.put(host, hostsBlockMap.get(host) + 1); - } else { - hostsBlockMap.put(host, 1); - } - } - } - - List> entries = new ArrayList>(hostsBlockMap.entrySet()); - Collections.sort(entries, new Comparator>() { - - @Override - public int compare(Map.Entry v1, Map.Entry v2) { - return v1.getValue().compareTo(v2.getValue()); - } - }); - - String[] hosts = new String[blkLocations[0].getHosts().length]; - - for (int i = 0; i < hosts.length; i++) { - Map.Entry entry = entries.get((entries.size() - 1) - i); - hosts[i] = entry.getKey(); - } - return new FileFragment(fragmentId, file, start, length, hosts); - } - - /** - * Get the minimum split size - * - * @return the minimum number of bytes that can be in a split - */ - public long getMinSplitSize() { - return conf.getLongVar(TajoConf.ConfVars.MINIMUM_SPLIT_SIZE); - } - - /** - * Get Disk Ids by Volume Bytes - */ - private int[] getDiskIds(VolumeId[] volumeIds) { - int[] diskIds = new int[volumeIds.length]; - for (int i = 0; i < volumeIds.length; i++) { - int diskId = -1; - if (volumeIds[i] != null && volumeIds[i].hashCode() > 0) { - diskId = volumeIds[i].hashCode() - zeroVolumeId.hashCode(); - } - diskIds[i] = diskId; - } - return diskIds; - } - - /** - * Generate the map of host and make them into Volume Ids. - * - */ - private Map> getVolumeMap(List frags) { - Map> volumeMap = new HashMap>(); - for (FileFragment frag : frags) { - String[] hosts = frag.getHosts(); - int[] diskIds = frag.getDiskIds(); - for (int i = 0; i < hosts.length; i++) { - Set volumeList = volumeMap.get(hosts[i]); - if (volumeList == null) { - volumeList = new HashSet(); - volumeMap.put(hosts[i], volumeList); - } - - if (diskIds.length > 0 && diskIds[i] > -1) { - volumeList.add(diskIds[i]); - } - } - } - - return volumeMap; - } - /** - * Generate the list of files and make them into FileSplits. - * - * @throws IOException - */ - public List getSplits(String tableName, TableMeta meta, Schema schema, Path... inputs) - throws IOException { - // generate splits' - - List splits = Lists.newArrayList(); - List volumeSplits = Lists.newArrayList(); - List blockLocations = Lists.newArrayList(); - - for (Path p : inputs) { - FileSystem fs = p.getFileSystem(conf); - ArrayList files = Lists.newArrayList(); - if (fs.isFile(p)) { - files.addAll(Lists.newArrayList(fs.getFileStatus(p))); - } else { - files.addAll(listStatus(p)); - } - - int previousSplitSize = splits.size(); - for (FileStatus file : files) { - Path path = file.getPath(); - long length = file.getLen(); - if (length > 0) { - // Get locations of blocks of file - BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); - boolean splittable = isSplittable(meta, schema, path, file); - if (blocksMetadataEnabled && fs instanceof DistributedFileSystem) { - - if (splittable) { - for (BlockLocation blockLocation : blkLocations) { - volumeSplits.add(makeSplit(tableName, path, blockLocation)); - } - blockLocations.addAll(Arrays.asList(blkLocations)); - - } else { // Non splittable - long blockSize = blkLocations[0].getLength(); - if (blockSize >= length) { - blockLocations.addAll(Arrays.asList(blkLocations)); - for (BlockLocation blockLocation : blkLocations) { - volumeSplits.add(makeSplit(tableName, path, blockLocation)); - } - } else { - splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); - } - } - - } else { - if (splittable) { - - long minSize = Math.max(getMinSplitSize(), 1); - - long blockSize = file.getBlockSize(); // s3n rest api contained block size but blockLocations is one - long splitSize = Math.max(minSize, blockSize); - long bytesRemaining = length; - - // for s3 - while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) { - int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); - splits.add(makeSplit(tableName, path, length - bytesRemaining, splitSize, - blkLocations[blkIndex].getHosts())); - bytesRemaining -= splitSize; - } - if (bytesRemaining > 0) { - int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); - splits.add(makeSplit(tableName, path, length - bytesRemaining, bytesRemaining, - blkLocations[blkIndex].getHosts())); - } - } else { // Non splittable - splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); - } - } - } else { - //for zero length files - splits.add(makeSplit(tableName, path, 0, length)); - } - } - if(LOG.isDebugEnabled()){ - LOG.debug("# of splits per partition: " + (splits.size() - previousSplitSize)); - } - } - - // Combine original fileFragments with new VolumeId information - setVolumeMeta(volumeSplits, blockLocations); - splits.addAll(volumeSplits); - LOG.info("Total # of splits: " + splits.size()); - return splits; - } - - private void setVolumeMeta(List splits, final List blockLocations) - throws IOException { - - int locationSize = blockLocations.size(); - int splitSize = splits.size(); - if (locationSize == 0 || splitSize == 0) return; - - if (locationSize != splitSize) { - // splits and locations don't match up - LOG.warn("Number of block locations not equal to number of splits: " - + "#locations=" + locationSize - + " #splits=" + splitSize); - return; - } - - DistributedFileSystem fs = (DistributedFileSystem)DistributedFileSystem.get(conf); - int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); - int blockLocationIdx = 0; - - Iterator iter = splits.iterator(); - while (locationSize > blockLocationIdx) { - - int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); - List locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); - //BlockStorageLocation containing additional volume location information for each replica of each block. - BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); - - for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { - iter.next().setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); - blockLocationIdx++; - } - } - LOG.info("# of splits with volumeId " + splitSize); - } - - private static class InvalidInputException extends IOException { - List errors; - public InvalidInputException(List errors) { - this.errors = errors; - } - - @Override - public String getMessage(){ - StringBuffer sb = new StringBuffer(); - int messageLimit = Math.min(errors.size(), 10); - for (int i = 0; i < messageLimit ; i ++) { - sb.append(errors.get(i).getMessage()).append("\n"); - } - - if(messageLimit < errors.size()) - sb.append("skipped .....").append("\n"); - - return sb.toString(); - } - } - private static final Class[] DEFAULT_SCANNER_PARAMS = { Configuration.class, Schema.class, TableMeta.class, - FileFragment.class + Fragment.class }; private static final Class[] DEFAULT_APPENDER_PARAMS = { @@ -773,14 +230,11 @@ public Class getScannerClass(CatalogProtos.StoreType storeTyp } public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { - if (fragment instanceof FileFragment) { - FileFragment fileFragment = (FileFragment)fragment; - if (fileFragment.getEndKey() == 0) { - Scanner scanner = new NullScanner(conf, schema, meta, fileFragment); - scanner.setTarget(target.toArray()); + if (fragment.isEmpty()) { + Scanner scanner = new NullScanner(conf, schema, meta, fragment); + scanner.setTarget(target.toArray()); - return scanner; - } + return scanner; } Scanner scanner; @@ -796,7 +250,7 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Sche public static synchronized SeekableScanner getSeekableScanner( TajoConf conf, TableMeta meta, Schema schema, FileFragment fragment, Schema target) throws IOException { - return (SeekableScanner)getStorageManager(conf, null).getScanner(meta, schema, fragment, target); + return (SeekableScanner)getStorageManager(conf, meta.getStoreType()).getScanner(meta, schema, fragment, target); } public static synchronized SeekableScanner getSeekableScanner( diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java index 816ae2564a..72472fced2 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java @@ -39,6 +39,7 @@ import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; import java.nio.ByteBuffer; @@ -63,7 +64,7 @@ public class AvroScanner extends FileScanner { */ public AvroScanner(Configuration conf, final org.apache.tajo.catalog.Schema schema, - final TableMeta meta, final FileFragment fragment) { + final TableMeta meta, final Fragment fragment) { super(conf, schema, meta, fragment); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index 6fe68411c7..dcd9f0a033 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -120,6 +120,7 @@ public void setDiskIds(int[] diskIds){ this.diskIds = diskIds; } + @Override public String getTableName() { return this.tableName; } @@ -136,10 +137,20 @@ public Long getStartKey() { return this.startOffset; } - public Long getEndKey() { + @Override + public String getKey() { + return this.uri.toString(); + } + + @Override + public long getLength() { return this.length; } + @Override + public boolean isEmpty() { + return this.length <= 0; + } /** * * The offset range of tablets MUST NOT be overlapped. @@ -169,7 +180,7 @@ public boolean equals(Object o) { FileFragment t = (FileFragment) o; if (getPath().equals(t.getPath()) && TUtil.checkEquals(t.getStartKey(), this.getStartKey()) - && TUtil.checkEquals(t.getEndKey(), this.getEndKey())) { + && TUtil.checkEquals(t.getLength(), this.getLength())) { return true; } } @@ -195,7 +206,7 @@ public Object clone() throws CloneNotSupportedException { public String toString() { return "\"fragment\": {\"id\": \""+ tableName +"\", \"path\": " +getPath() + "\", \"start\": " + this.getStartKey() + ",\"length\": " - + getEndKey() + "}" ; + + getLength() + "}" ; } public FragmentProto getProto() { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/Fragment.java b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/Fragment.java index 3f9c16012a..ac43197296 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/Fragment.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/Fragment.java @@ -28,4 +28,12 @@ public interface Fragment extends ProtoObject { @Override public abstract FragmentProto getProto(); + + public abstract long getLength(); + + public abstract String getKey(); + + public String[] getHosts(); + + public abstract boolean isEmpty(); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java index 36b89b8ac3..2f8efcffca 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java @@ -23,7 +23,7 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.storage.FileScanner; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; @@ -42,7 +42,7 @@ public class ParquetScanner extends FileScanner { * @param fragment */ public ParquetScanner(Configuration conf, final Schema schema, - final TableMeta meta, final FileFragment fragment) { + final TableMeta meta, final Fragment fragment) { super(conf, schema, meta, fragment); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java index e5507ad81f..0e5c0e97b3 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java @@ -39,6 +39,7 @@ import org.apache.tajo.datum.NullDatum; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import java.io.Closeable; import java.io.*; @@ -1176,12 +1177,12 @@ private static class SelectedColumn { private SerializerDeserializer serde; public RCFileScanner(Configuration conf, final Schema schema, final TableMeta meta, - final FileFragment fragment) throws IOException { + final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); conf.setInt("io.file.buffer.size", 4096); //TODO remove - startOffset = fragment.getStartKey(); - endOffset = startOffset + fragment.getEndKey(); + startOffset = this.fragment.getStartKey(); + endOffset = startOffset + this.fragment.getLength(); start = 0; } @@ -1651,7 +1652,7 @@ public float getProgress() { return 0.0f; } else { //if scanner read the header, filePos moved to zero - return Math.min(1.0f, (float)(Math.max(filePos - startOffset, 0)) / (float)(fragment.getEndKey())); + return Math.min(1.0f, (float)(Math.max(filePos - startOffset, 0)) / (float)(fragment.getLength())); } } catch (IOException e) { LOG.error(e.getMessage(), e); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java index b0ef67d97d..74563fff10 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java @@ -32,7 +32,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.BytesUtils; import java.io.IOException; @@ -71,7 +71,7 @@ public class SequenceFileScanner extends FileScanner { private Writable EMPTY_KEY; - public SequenceFileScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException { + public SequenceFileScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { super(conf, schema, meta, fragment); } @@ -96,7 +96,7 @@ public void init() throws IOException { this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0); this.start = fragment.getStartKey(); - this.end = start + fragment.getEndKey(); + this.end = start + fragment.getLength(); if (fragment.getStartKey() > reader.getPosition()) reader.sync(this.start); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java index 36a589aade..6bca48b791 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java @@ -30,7 +30,7 @@ import org.apache.tajo.storage.FileScanner; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.trevni.ColumnFileReader; import org.apache.trevni.ColumnValues; import org.apache.trevni.avro.HadoopInput; @@ -45,9 +45,9 @@ public class TrevniScanner extends FileScanner { private int [] projectionMap; private ColumnValues [] columns; - public TrevniScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException { + public TrevniScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { super(conf, schema, meta, fragment); - reader = new ColumnFileReader(new HadoopInput(fragment.getPath(), conf)); + reader = new ColumnFileReader(new HadoopInput(this.fragment.getPath(), conf)); } @Override diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index 212f374e8c..8f79f4bc95 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -124,7 +124,7 @@ public void testSplitCompressionData() throws IOException { meta.putOption("compression.codec", BZip2Codec.class.getCanonicalName()); Path tablePath = new Path(testDir, "SplitCompression"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); @@ -156,7 +156,7 @@ public void testSplitCompressionData() throws IOException { tablets[0] = new FileFragment("SplitCompression", tablePath, 0, randomNum); tablets[1] = new FileFragment("SplitCompression", tablePath, randomNum, (fileLen - randomNum)); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema); assertTrue(scanner.isSplittable()); scanner.init(); int tupleCnt = 0; @@ -166,7 +166,7 @@ public void testSplitCompressionData() throws IOException { } scanner.close(); - scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); + scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[1], schema); assertTrue(scanner.isSplittable()); scanner.init(); while ((tuple = scanner.next()) != null) { @@ -191,7 +191,7 @@ private void storageCompressionTest(StoreType storeType, Class splits = sm.getSplits("table", meta, schema, path); + List splits = sm.getSplits("table", meta, schema, path); int splitSize = (int) Math.ceil(fileStatus.getLen() / (double) fileStatus.getBlockSize()); assertEquals(splitSize, splits.size()); - for (FileFragment fragment : splits) { - assertTrue(fragment.getEndKey() <= fileStatus.getBlockSize()); + for (Fragment fragment : splits) { + assertTrue(fragment.getLength() <= fileStatus.getBlockSize()); } } } diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index c1a96a5d08..1b80237276 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -95,7 +96,7 @@ public void setup() throws Exception { conf.setStrings("tajo.storage.projectable-scanner", "rcfile", "trevni", "parquet", "avro"); testDir = CommonTestingUtil.getTestDir(TEST_PATH); fs = testDir.getFileSystem(conf); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); } @Test @@ -115,7 +116,7 @@ public void testMultipleFiles() throws IOException { } Path table1Path = new Path(testDir, storeType + "_1.data"); - Appender appender1 = StorageManager.getStorageManager(conf).getAppender(meta, schema, table1Path); + Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, table1Path); appender1.enableStats(); appender1.init(); int tupleNum = 10000; @@ -137,7 +138,7 @@ public void testMultipleFiles() throws IOException { } Path table2Path = new Path(testDir, storeType + "_2.data"); - Appender appender2 = StorageManager.getStorageManager(conf).getAppender(meta, schema, table2Path); + Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, table2Path); appender2.enableStats(); appender2.init(); @@ -159,7 +160,7 @@ public void testMultipleFiles() throws IOException { FileStatus status1 = fs.getFileStatus(table1Path); FileStatus status2 = fs.getFileStatus(table2Path); - FileFragment[] fragment = new FileFragment[2]; + Fragment[] fragment = new Fragment[2]; fragment[0] = new FileFragment("tablet1", table1Path, 0, status1.getLen()); fragment[1] = new FileFragment("tablet1", table2Path, 0, status2.getLen()); @@ -167,7 +168,7 @@ public void testMultipleFiles() throws IOException { targetSchema.addColumn(schema.getColumn(0)); targetSchema.addColumn(schema.getColumn(2)); - Scanner scanner = new MergeScanner(conf, schema, meta, TUtil.newList(fragment), targetSchema); + Scanner scanner = new MergeScanner(conf, schema, meta, TUtil.newList(fragment), targetSchema); assertEquals(isProjectableStorage(meta.getStoreType()), scanner.isProjectable()); scanner.init(); diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java index c3d4992e3d..13aeef64b1 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java @@ -32,6 +32,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.CommonTestingUtil; import org.junit.After; import org.junit.Before; @@ -56,7 +57,7 @@ public void setUp() throws Exception { conf = new TajoConf(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); fs = testDir.getFileSystem(conf); - sm = StorageManager.getStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf, testDir); } @After @@ -83,14 +84,14 @@ public final void testGetScannerAndAppender() throws IOException { Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv"); fs.mkdirs(path.getParent()); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, path); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, path); appender.init(); for(Tuple t : tuples) { appender.addTuple(t); } appender.close(); - Scanner scanner = StorageManager.getStorageManager(conf).getFileScanner(meta, schema, path); + Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(meta, schema, path); scanner.init(); int i=0; while(scanner.next() != null) { @@ -124,7 +125,7 @@ public void testGetSplit() throws Exception { } assertTrue(fs.exists(tablePath)); - StorageManager sm = StorageManager.getStorageManager(new TajoConf(conf), tablePath); + FileStorageManager sm = StorageManager.getFileStorageManager(new TajoConf(conf), tablePath); Schema schema = new Schema(); schema.addColumn("id", Type.INT4); @@ -132,19 +133,19 @@ public void testGetSplit() throws Exception { schema.addColumn("name",Type.TEXT); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); - List splits = Lists.newArrayList(); + List splits = Lists.newArrayList(); // Get FileFragments in partition batch splits.addAll(sm.getSplits("data", meta, schema, partitions.toArray(new Path[partitions.size()]))); assertEquals(testCount, splits.size()); // -1 is unknown volumeId - assertEquals(-1, splits.get(0).getDiskIds()[0]); + assertEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]); splits.clear(); splits.addAll(sm.getSplits("data", meta, schema, partitions.subList(0, partitions.size() / 2).toArray(new Path[partitions.size() / 2]))); assertEquals(testCount / 2, splits.size()); assertEquals(1, splits.get(0).getHosts().length); - assertEquals(-1, splits.get(0).getDiskIds()[0]); + assertEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]); fs.close(); } finally { cluster.shutdown(); @@ -176,7 +177,7 @@ public void testGetSplitWithBlockStorageLocationsBatching() throws Exception { DFSTestUtil.createFile(fs, tmpFile, 10, (short) 2, 0xDEADDEADl); } assertTrue(fs.exists(tablePath)); - StorageManager sm = StorageManager.getStorageManager(new TajoConf(conf), tablePath); + FileStorageManager sm = StorageManager.getFileStorageManager(new TajoConf(conf), tablePath); Schema schema = new Schema(); schema.addColumn("id", Type.INT4); @@ -184,13 +185,13 @@ public void testGetSplitWithBlockStorageLocationsBatching() throws Exception { schema.addColumn("name", Type.TEXT); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); - List splits = Lists.newArrayList(); + List splits = Lists.newArrayList(); splits.addAll(sm.getSplits("data", meta, schema, tablePath)); assertEquals(testCount, splits.size()); assertEquals(2, splits.get(0).getHosts().length); - assertEquals(2, splits.get(0).getDiskIds().length); - assertNotEquals(-1, splits.get(0).getDiskIds()[0]); + assertEquals(2, ((FileFragment)splits.get(0)).getDiskIds().length); + assertNotEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]); fs.close(); } finally { cluster.shutdown(); diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java index ef5388cde4..4e80fa91c6 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -139,7 +139,7 @@ public void testSplitable() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "Splitable.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); int tupleNum = 10000; @@ -163,7 +163,7 @@ public void testSplitable() throws IOException { tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum); tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum)); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema); assertTrue(scanner.isSplittable()); scanner.init(); int tupleCnt = 0; @@ -172,7 +172,7 @@ public void testSplitable() throws IOException { } scanner.close(); - scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); + scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[1], schema); assertTrue(scanner.isSplittable()); scanner.init(); while (scanner.next() != null) { @@ -193,7 +193,7 @@ public void testRCFileSplitable() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "Splitable.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); int tupleNum = 10000; @@ -217,7 +217,7 @@ public void testRCFileSplitable() throws IOException { tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum); tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum)); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema); assertTrue(scanner.isSplittable()); scanner.init(); int tupleCnt = 0; @@ -226,7 +226,7 @@ public void testRCFileSplitable() throws IOException { } scanner.close(); - scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); + scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[1], schema); assertTrue(scanner.isSplittable()); scanner.init(); while (scanner.next() != null) { @@ -253,7 +253,7 @@ public void testProjection() throws IOException { } Path tablePath = new Path(testDir, "testProjection.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); int tupleNum = 10000; VTuple vTuple; @@ -273,7 +273,7 @@ public void testProjection() throws IOException { Schema target = new Schema(); target.addColumn("age", Type.INT8); target.addColumn("score", Type.FLOAT4); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment, target); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment, target); scanner.init(); int tupleCnt = 0; Tuple tuple; @@ -321,7 +321,7 @@ public void testVariousTypes() throws IOException { } Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); QueryId queryid = new QueryId("12345", 5); @@ -349,7 +349,7 @@ public void testVariousTypes() throws IOException { FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); Tuple retrieved; @@ -391,7 +391,7 @@ public void testNullHandlingTypes() throws IOException { } Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); QueryId queryid = new QueryId("12345", 5); @@ -432,7 +432,7 @@ public void testNullHandlingTypes() throws IOException { FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); Tuple retrieved; @@ -476,7 +476,7 @@ public void testRCFileTextSerializeDeserialize() throws IOException { meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); @@ -507,7 +507,7 @@ public void testRCFileTextSerializeDeserialize() throws IOException { assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); Tuple retrieved; @@ -545,7 +545,7 @@ public void testRCFileBinarySerializeDeserialize() throws IOException { meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); @@ -576,7 +576,7 @@ public void testRCFileBinarySerializeDeserialize() throws IOException { assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); Tuple retrieved; @@ -614,7 +614,7 @@ public void testSequenceFileTextSerializeDeserialize() throws IOException { meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); @@ -645,7 +645,7 @@ public void testSequenceFileTextSerializeDeserialize() throws IOException { assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); assertTrue(scanner instanceof SequenceFileScanner); @@ -687,7 +687,7 @@ public void testSequenceFileBinarySerializeDeserialize() throws IOException { meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); @@ -718,7 +718,7 @@ public void testSequenceFileBinarySerializeDeserialize() throws IOException { assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen()); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); assertTrue(scanner instanceof SequenceFileScanner); @@ -748,7 +748,7 @@ public void testTime() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(storeType, options); Path tablePath = new Path(testDir, "testTime.data"); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(3); @@ -763,7 +763,7 @@ public void testTime() throws IOException { FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); Tuple retrieved; @@ -789,8 +789,8 @@ public void testSeekableScanner() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "Seekable.data"); - FileAppender appender = (FileAppender) StorageManager.getStorageManager(conf).getAppender(meta, schema, - tablePath); + FileAppender appender = (FileAppender) StorageManager.getFileStorageManager(conf).getAppender(meta, schema, + tablePath); appender.enableStats(); appender.init(); int tupleNum = 100000; @@ -831,18 +831,18 @@ public void testSeekableScanner() throws IOException { long readBytes = 0; long readRows = 0; for (long offset : offsets) { - scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, - new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema); + scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, + new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema); scanner.init(); while (scanner.next() != null) { - tupleCnt++; + tupleCnt++; } scanner.close(); if (statsable) { - readBytes += scanner.getInputStats().getNumBytes(); - readRows += scanner.getInputStats().getNumRows(); + readBytes += scanner.getInputStats().getNumBytes(); + readRows += scanner.getInputStats().getNumRows(); } prevOffset = offset; } diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java index bf13f3c13e..ef27e455f1 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java @@ -89,7 +89,7 @@ public void testFindValue() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testFindValue_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for (int i = 0; i < TUPLE_NUM; i++) { @@ -177,7 +177,7 @@ public void testBuildIndexWithAppender() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testBuildIndexWithAppender_" + storeType); - FileAppender appender = (FileAppender) StorageManager.getStorageManager(conf).getAppender(meta, schema, + FileAppender appender = (FileAppender) StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); @@ -256,7 +256,7 @@ public void testFindOmittedValue() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = StorageUtil.concatPath(testDir, "testFindOmittedValue_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for (int i = 0; i < TUPLE_NUM; i += 2) { @@ -326,7 +326,7 @@ public void testFindNextKeyValue() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testFindNextKeyValue_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for (int i = 0; i < TUPLE_NUM; i++) { @@ -416,7 +416,7 @@ public void testFindNextKeyOmittedValue() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testFindNextKeyOmittedValue_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for (int i = 0; i < TUPLE_NUM; i += 2) { @@ -495,7 +495,7 @@ public void testFindMinValue() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testFindMinValue" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; @@ -578,7 +578,7 @@ public void testMinMax() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testMinMax_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for (int i = 5; i < TUPLE_NUM; i += 2) { @@ -682,7 +682,7 @@ public void testConcurrentAccess() throws IOException, InterruptedException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testConcurrentAccess_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; @@ -762,7 +762,7 @@ public void testFindValueDescOrder() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testFindValueDescOrder_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; @@ -853,7 +853,7 @@ public void testFindNextKeyValueDescOrder() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testFindNextKeyValueDescOrder_" + storeType); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index 78b16c3804..5f758a2729 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -78,7 +78,7 @@ public void testFindValueInSingleCSV() throws IOException { Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv"); fs.mkdirs(tablePath.getParent()); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for (int i = 0; i < TUPLE_NUM; i++) { @@ -167,7 +167,7 @@ public void testFindNextKeyValueInSingleCSV() throws IOException { Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV", "table1.csv"); fs.mkdirs(tablePath.getParent()); - Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for(int i = 0 ; i < TUPLE_NUM; i ++ ) { From ec11bef177e5b95ac8842df465ff111759cec501 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Fri, 24 Oct 2014 19:26:52 +0900 Subject: [PATCH 02/29] TAJO-1127: Implements HBaseStorageManager Implements HBaseStorageManager and HBaseScanner. --- .../org/apache/tajo/catalog/CatalogUtil.java | 2 + .../src/main/proto/CatalogProtos.proto | 4 +- .../java/org/apache/tajo/TajoConstants.java | 1 + .../java/org/apache/tajo/conf/TajoConf.java | 2 + tajo-core/pom.xml | 28 ++ .../apache/tajo/engine/parser/SQLParser.g4 | 10 +- .../tajo/engine/parser/SQLAnalyzer.java | 18 +- .../engine/planner/PhysicalPlannerImpl.java | 21 +- .../tajo/engine/planner/PlannerUtil.java | 193 ++++++++- .../engine/planner/physical/SeqScanExec.java | 6 +- .../rewrite/PartitionedTableRewriter.java | 47 +-- .../DefaultFragmentScheduleAlgorithm.java | 2 +- .../GreedyFragmentScheduleAlgorithm.java | 9 +- .../apache/tajo/master/LazyTaskScheduler.java | 17 +- .../tajo/master/querymaster/QueryUnit.java | 2 +- .../org/apache/tajo/HBaseTestClusterUtil.java | 155 +++++++ .../org/apache/tajo/TajoTestingCluster.java | 16 + .../tajo/engine/planner/TestPlannerUtil.java | 2 +- .../planner/physical/TestBSTIndexExec.java | 3 +- .../tajo/engine/query/TestHBaseTable.java | 325 ++++++++++++++ .../testBinaryMappedQuery.result | 81 ++++ .../testIndexPredication.result | 38 ++ .../testRowFieldSelectQuery.result | 88 ++++ .../testSimpleSelectQuery.result | 88 ++++ tajo-storage/pom.xml | 7 + .../tajo/storage/FileStorageManager.java | 12 +- .../apache/tajo/storage/IndexPredication.java | 61 +++ .../apache/tajo/storage/StorageManager.java | 15 +- .../tajo/storage/fragment/FileFragment.java | 2 + .../storage/fragment/FragmentConvertor.java | 26 +- .../tajo/storage/hbase/ColumnMapping.java | 171 ++++++++ .../tajo/storage/hbase/HBaseAppender.java | 62 +++ .../HBaseBinarySerializerDeserializer.java | 104 +++++ .../tajo/storage/hbase/HBaseFragment.java | 189 +++++++++ .../tajo/storage/hbase/HBaseLazyTuple.java | 318 ++++++++++++++ .../tajo/storage/hbase/HBaseScanner.java | 242 +++++++++++ .../storage/hbase/HBaseStorageManager.java | 399 ++++++++++++++++++ .../HBaseTextSerializerDeserializer.java | 75 ++++ .../tajo/storage/hbase/RowKeyMapping.java | 40 ++ .../main/proto/StorageFragmentProtos.proto | 35 ++ .../src/main/resources/storage-default.xml | 67 +-- 41 files changed, 2824 insertions(+), 159 deletions(-) create mode 100644 tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java create mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testBinaryMappedQuery.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/IndexPredication.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java create mode 100644 tajo-storage/src/main/proto/StorageFragmentProtos.proto diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 7486eff968..30dda00e04 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -274,6 +274,8 @@ public static StoreType getStoreType(final String typeStr) { return StoreType.SEQUENCEFILE; } else if (typeStr.equalsIgnoreCase(StoreType.AVRO.name())) { return StoreType.AVRO; + } else if (typeStr.equalsIgnoreCase(StoreType.HBASE.name())) { + return StoreType.HBASE; } else { return null; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index db402ae359..60106065e4 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -36,6 +36,7 @@ enum StoreType { PARQUET = 7; SEQUENCEFILE = 8; AVRO = 9; + HBASE = 10; } enum OrderType { @@ -68,7 +69,8 @@ message SchemaProto { message FragmentProto { required string id = 1; - required bytes contents = 2; + required string storeType = 2; + required bytes contents = 3; } message FileFragmentProto { diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java index 08909b4f46..de09f098fd 100644 --- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java @@ -39,6 +39,7 @@ public class TajoConstants { public static final String SYSTEM_HA_BACKUP_DIR_NAME = "backup"; public static final int UNKNOWN_ROW_NUMBER = -1; + public static final int UNKNOWN_LENGTH = -1; private TajoConstants() {} } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 181ef2e536..1d03130cea 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -279,6 +279,8 @@ public static enum ConfVars implements ConfigKey { METRICS_PROPERTY_FILENAME("tajo.metrics.property.file", "tajo-metrics.properties"), // Misc ------------------------------------------------------------------- + // Fragment + FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH("tajo.fragment.alternative.unknown.length", (long)(512 * 1024 * 1024)), // Geo IP GEOIP_DATA("tajo.function.geoip-database-location", ""), diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index ca302664ae..bf3b7abc8c 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -34,6 +34,7 @@ true true 3.0.1 + 0.98.6.1-hadoop2 @@ -392,6 +393,33 @@ gmetric4j 1.0.3 + + org.apache.hbase + hbase-server + ${hbase.version} + test-jar + test + + + org.apache.hbase + hbase-server + ${hbase.version} + test + + + org.apache.hbase + hbase-hadoop-compat + ${hbase.version} + test-jar + test + + + org.apache.hbase + hbase-hadoop2-compat + ${hbase.version} + test-jar + test + diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index beba248be9..9b63a244e7 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -89,11 +89,11 @@ if_exists ; create_table_statement - : CREATE EXTERNAL TABLE (if_not_exists)? table_name table_elements USING file_type=identifier - (param_clause)? (table_partitioning_clauses)? (LOCATION path=Character_String_Literal) - | CREATE TABLE (if_not_exists)? table_name table_elements (USING file_type=identifier)? + : CREATE EXTERNAL TABLE (if_not_exists)? table_name table_elements USING storage_type=identifier + (param_clause)? (table_partitioning_clauses)? (LOCATION path=Character_String_Literal)? + | CREATE TABLE (if_not_exists)? table_name table_elements (USING storage_type=identifier)? (param_clause)? (table_partitioning_clauses)? (AS query_expression)? - | CREATE TABLE (if_not_exists)? table_name (USING file_type=identifier)? + | CREATE TABLE (if_not_exists)? table_name (USING storage_type=identifier)? (param_clause)? (table_partitioning_clauses)? AS query_expression | CREATE TABLE (if_not_exists)? table_name LIKE like_table_name=table_name ; @@ -1559,7 +1559,7 @@ null_ordering insert_statement : INSERT (OVERWRITE)? INTO table_name (LEFT_PAREN column_name_list RIGHT_PAREN)? query_expression - | INSERT (OVERWRITE)? INTO LOCATION path=Character_String_Literal (USING file_type=identifier (param_clause)?)? query_expression + | INSERT (OVERWRITE)? INTO LOCATION path=Character_String_Literal (USING storage_type=identifier (param_clause)?)? query_expression ; /* diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 580ec61e3c..93a76d11a2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -29,6 +29,7 @@ import org.apache.tajo.algebra.LiteralValue.LiteralType; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.engine.parser.SQLParser.*; +import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.StringUtils; @@ -62,6 +63,7 @@ public Expr parse(String sql) { try { context = parser.sql(); } catch (SQLParseError e) { + e.printStackTrace(); throw new SQLSyntaxError(e); } return visitSql(context); @@ -1162,12 +1164,14 @@ public Expr visitCreate_table_statement(SQLParser.Create_table_statementContext createTable.setExternal(); ColumnDefinition[] elements = getDefinitions(ctx.table_elements()); - String fileType = ctx.file_type.getText(); - String path = stripQuote(ctx.path.getText()); - + String storageType = ctx.storage_type.getText(); createTable.setTableElements(elements); - createTable.setStorageType(fileType); - createTable.setLocation(path); + createTable.setStorageType(storageType); + + if (PlannerUtil.isFileStorageType(storageType)) { + String path = stripQuote(ctx.path.getText()); + createTable.setLocation(path); + } } else { if (checkIfExist(ctx.table_elements())) { ColumnDefinition[] elements = getDefinitions(ctx.table_elements()); @@ -1175,7 +1179,7 @@ public Expr visitCreate_table_statement(SQLParser.Create_table_statementContext } if (checkIfExist(ctx.USING())) { - String fileType = ctx.file_type.getText(); + String fileType = ctx.storage_type.getText(); createTable.setStorageType(fileType); } @@ -1449,7 +1453,7 @@ public Expr visitInsert_statement(SQLParser.Insert_statementContext ctx) { insertExpr.setLocation(stripQuote(ctx.path.getText())); if (ctx.USING() != null) { - insertExpr.setStorageType(ctx.file_type.getText()); + insertExpr.setStorageType(ctx.storage_type.getText()); if (ctx.param_clause() != null) { insertExpr.setParams(escapeTableMeta(getParams(ctx.param_clause()))); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index d4c5b70acb..7f80a54f23 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -29,11 +29,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.logical.*; @@ -243,15 +245,22 @@ private PhysicalExec createPlanRecursive(TaskAttemptContext ctx, LogicalNode log } } + public static long getFragmentLength(TajoConf conf, Fragment fragment) { + if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { + return conf.getLongVar(ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); + } else { + return fragment.getLength(); + } + } + @VisibleForTesting public long estimateSizeRecursive(TaskAttemptContext ctx, String [] tableIds) throws IOException { long size = 0; for (String tableId : tableIds) { - // TODO - CSV is a hack. - List fragments = FragmentConvertor.convert(ctx.getConf(), CatalogProtos.StoreType.CSV, - ctx.getTables(tableId)); - for (FileFragment frag : fragments) { - size += frag.getLength(); + FragmentProto[] fragmentProtos = ctx.getTables(tableId); + List fragments = FragmentConvertor.convert(ctx.getConf(), fragmentProtos); + for (Fragment frag : fragments) { + size += getFragmentLength(ctx.getConf(), frag); } } return size; @@ -1179,7 +1188,7 @@ public PhysicalExec createIndexScanExec(TaskAttemptContext ctx, FragmentProto [] fragmentProtos = ctx.getTables(annotation.getTableName()); List fragments = - FragmentConvertor.convert(ctx.getConf(), ctx.getDataChannel().getStoreType(), fragmentProtos); + FragmentConvertor.convert(ctx.getConf(), fragmentProtos); String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys()); Path indexPath = new Path( diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java index 02e5c2de71..6365e9e5b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java @@ -35,15 +35,18 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.exception.InvalidQueryException; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.storage.FileStorageManager; +import org.apache.tajo.storage.IndexPredication; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; +import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.io.IOException; @@ -180,9 +183,20 @@ public static Collection getRelationLineageWithinQueryBlock(LogicalPlan return visitor.getFoundRelations(); } + public static boolean isFileStorageType(String storageType) { + if (storageType.equalsIgnoreCase("hbase")) { + return false; + } else { + return true; + } + } + public static boolean isFileStorageType(StoreType storageType) { - //Currently all storage type are a file storage. - return true; + if (storageType== StoreType.HBASE) { + return false; + } else { + return true; + } } public static class RelationFinderVisitor extends BasicLogicalPlanVisitor { @@ -892,4 +906,179 @@ private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List getIndexPredications(StorageManager storageHandler, + TableDesc tableDesc, ScanNode scan) throws IOException { + List indexPredications = new ArrayList(); + Column[] indexableColumns = storageHandler.getIndexableColumns(tableDesc); + if (indexableColumns != null && indexableColumns.length == 1) { + // Currently supports only single index column. + Set indexablePredicateSet = PlannerUtil.findIndexablePredicateSet(scan, indexableColumns, true); + Pair indexPredicationValues = PlannerUtil.getIndexablePredicateValue(indexablePredicateSet); + if (indexPredicationValues != null) { + IndexPredication indexPredication = new IndexPredication(); + indexPredication.setColumn(indexableColumns[0]); + indexPredication.setColumnId(tableDesc.getLogicalSchema().getColumnId(indexableColumns[0].getQualifiedName())); + indexPredication.setStartValue(indexPredicationValues.getFirst()); + indexPredication.setStopValue(indexPredicationValues.getSecond()); + + indexPredications.add(indexPredication); + } + } + return indexPredications; + } + + public static Set findIndexablePredicateSet(ScanNode scanNode, Column[] indexableColumns, + boolean preserveFoundPredicate) throws IOException { + Set indexablePredicateSet = Sets.newHashSet(); + // if a query statement has a search condition, try to find indexable predicates + if (indexableColumns != null && scanNode.hasQual()) { + EvalNode[] conjunctiveForms = AlgebraicUtil.toConjunctiveNormalFormArray(scanNode.getQual()); + Set remainExprs = Sets.newHashSet(conjunctiveForms); + + // add qualifier to schema for qual + for (Column column : indexableColumns) { + for (EvalNode simpleExpr : conjunctiveForms) { + if (checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) { + indexablePredicateSet.add(simpleExpr); + } + } + } + + // Partitions which are not matched to the partition filter conditions are pruned immediately. + // So, the partition filter conditions are not necessary later, and they are removed from + // original search condition for simplicity and efficiency. + remainExprs.removeAll(indexablePredicateSet); + if (!preserveFoundPredicate) { + if (remainExprs.isEmpty()) { + scanNode.setQual(null); + } else { + scanNode.setQual( + AlgebraicUtil.createSingletonExprFromCNF(remainExprs.toArray(new EvalNode[remainExprs.size()]))); + } + } + } + + return indexablePredicateSet; + } + + public static boolean checkIfIndexablePredicateOnTargetColumn(EvalNode evalNode, Column targetColumn) { + if (checkIfIndexablePredicate(evalNode) || checkIfDisjunctiveButOneVariable(evalNode)) { + Set variables = EvalTreeUtil.findUniqueColumns(evalNode); + // if it contains only single variable matched to a target column + return variables.size() == 1 && variables.contains(targetColumn); + } else { + return false; + } + } + + /** + * + * @param evalNode The expression to be checked + * @return true if an disjunctive expression, consisting of indexable expressions + */ + public static boolean checkIfDisjunctiveButOneVariable(EvalNode evalNode) { + if (evalNode.getType() == EvalType.OR) { + BinaryEval orEval = (BinaryEval) evalNode; + boolean indexable = + checkIfIndexablePredicate(orEval.getLeftExpr()) && + checkIfIndexablePredicate(orEval.getRightExpr()); + + boolean sameVariable = + EvalTreeUtil.findUniqueColumns(orEval.getLeftExpr()) + .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr())); + + return indexable && sameVariable; + } else { + return false; + } + } + + /** + * Check if an expression consists of one variable and one constant and + * the expression is a comparison operator. + * + * @param evalNode The expression to be checked + * @return true if an expression consists of one variable and one constant + * and the expression is a comparison operator. Other, false. + */ + public static boolean checkIfIndexablePredicate(EvalNode evalNode) { + // TODO - LIKE with a trailing wild-card character and IN with an array can be indexable + return AlgebraicUtil.containSingleVar(evalNode) && AlgebraicUtil.isIndexableOperator(evalNode); + } + + public static Pair getIndexablePredicateValue(Set indexablePredicateSet) { + if (indexablePredicateSet.size() > 2) { + return null; + } + + Datum startDatum = null; + Datum endDatum = null; + for (EvalNode evalNode: indexablePredicateSet) { + if (evalNode instanceof BinaryEval) { + BinaryEval binaryEval = (BinaryEval) evalNode; + EvalNode left = binaryEval.getLeftExpr(); + EvalNode right = binaryEval.getRightExpr(); + + Datum constValue = null; + if (left.getType() == EvalType.CONST) { + constValue = ((ConstEval) left).getValue(); + } else if (right.getType() == EvalType.CONST) { + constValue = ((ConstEval) right).getValue(); + } + + if (evalNode.getType() == EvalType.EQUAL || + evalNode.getType() == EvalType.GEQ || + evalNode.getType() == EvalType.GTH) { + if (startDatum != null) { + if (constValue.compareTo(startDatum) < 0) { + startDatum = constValue; + } + } else { + startDatum = constValue; + } + } + + if (evalNode.getType() == EvalType.EQUAL || + evalNode.getType() == EvalType.LEQ || + evalNode.getType() == EvalType.LTH) { + if (endDatum != null) { + if (constValue.compareTo(endDatum) > 0) { + endDatum = constValue; + } + } else { + endDatum = constValue; + } + } + } else if (evalNode instanceof BetweenPredicateEval) { + BetweenPredicateEval betweenEval = (BetweenPredicateEval) evalNode; + if (betweenEval.getBegin().getType() == EvalType.CONST) { + Datum value = ((ConstEval) betweenEval.getBegin()).getValue(); + if (startDatum != null) { + if (value.compareTo(startDatum) < 0) { + startDatum = value; + } + } else { + startDatum = value; + } + } + if (betweenEval.getEnd().getType() == EvalType.CONST) { + Datum value = ((ConstEval) betweenEval.getEnd()).getValue(); + if (endDatum != null) { + if (value.compareTo(endDatum) > 0) { + endDatum = value; + } + } else { + endDatum = value; + } + } + } + } + + if (startDatum != null || endDatum != null) { + return new Pair(startDatum, endDatum); + } else { + return null; + } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index f881a007f2..06c361f7c6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -80,8 +80,7 @@ public SeqScanExec(TaskAttemptContext context, ScanNode plan, String pathNameKey = ""; if (fragments != null) { for (FragmentProto f : fragments) { - Fragment fragement = FragmentConvertor.convert( - context.getConf(), plan.getTableDesc().getMeta().getStoreType(), f); + Fragment fragement = FragmentConvertor.convert(context.getConf(), f); pathNameKey += fragement.getKey(); } } @@ -215,8 +214,7 @@ private void initScanner(Schema projected) throws IOException { if (fragments != null) { if (fragments.length > 1) { this.scanner = new MergeScanner(context.getConf(), plan.getPhysicalSchema(), plan.getTableDesc().getMeta(), - FragmentConvertor.convert(context.getConf(), plan.getTableDesc().getMeta().getStoreType(), - fragments), projected + FragmentConvertor.convert(context.getConf(), fragments), projected ); } else { StorageManager storageManager = StorageManager.getStorageManager( diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java index 666c5fc1ef..328263080c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java @@ -240,7 +240,7 @@ public String toString() { paritionValuesSchema.setQualifier(scanNode.getCanonicalName()); for (Column column : paritionValuesSchema.getColumns()) { for (EvalNode simpleExpr : conjunctiveForms) { - if (checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) { + if (PlannerUtil.checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) { indexablePredicateSet.add(simpleExpr); } } @@ -266,51 +266,6 @@ public String toString() { } } - private boolean checkIfIndexablePredicateOnTargetColumn(EvalNode evalNode, Column targetColumn) { - if (checkIfIndexablePredicate(evalNode) || checkIfDisjunctiveButOneVariable(evalNode)) { - Set variables = EvalTreeUtil.findUniqueColumns(evalNode); - // if it contains only single variable matched to a target column - return variables.size() == 1 && variables.contains(targetColumn); - } else { - return false; - } - } - - /** - * Check if an expression consists of one variable and one constant and - * the expression is a comparison operator. - * - * @param evalNode The expression to be checked - * @return true if an expression consists of one variable and one constant - * and the expression is a comparison operator. Other, false. - */ - private boolean checkIfIndexablePredicate(EvalNode evalNode) { - // TODO - LIKE with a trailing wild-card character and IN with an array can be indexable - return AlgebraicUtil.containSingleVar(evalNode) && AlgebraicUtil.isIndexableOperator(evalNode); - } - - /** - * - * @param evalNode The expression to be checked - * @return true if an disjunctive expression, consisting of indexable expressions - */ - private boolean checkIfDisjunctiveButOneVariable(EvalNode evalNode) { - if (evalNode.getType() == EvalType.OR) { - BinaryEval orEval = (BinaryEval) evalNode; - boolean indexable = - checkIfIndexablePredicate(orEval.getLeftExpr()) && - checkIfIndexablePredicate(orEval.getRightExpr()); - - boolean sameVariable = - EvalTreeUtil.findUniqueColumns(orEval.getLeftExpr()) - .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr())); - - return indexable && sameVariable; - } else { - return false; - } - } - private void updateTableStat(PartitionedTableScanNode scanNode) throws PlanningException { if (scanNode.getInputPaths().length > 0) { try { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java b/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java index 6a2a705478..406550d90d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultFragmentScheduleAlgorithm.java @@ -87,7 +87,7 @@ public void addFragment(FragmentPair fragmentPair) { diskIds = ((FileFragment)fragmentPair.getLeftFragment()).getDiskIds(); } for (int i = 0; i < hosts.length; i++) { - addFragment(hosts[i], diskIds[i], fragmentPair); + addFragment(hosts[i], diskIds != null ? diskIds[i] : -1, fragmentPair); } fragmentNum++; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java b/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java index 37983996af..56cf8e52f8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GreedyFragmentScheduleAlgorithm.java @@ -107,7 +107,7 @@ public void addFragment(FragmentPair fragmentPair) { diskIds = ((FileFragment)fragmentPair.getLeftFragment()).getDiskIds(); } for (int i = 0; i < hosts.length; i++) { - addFragment(hosts[i], diskIds[i], fragmentPair); + addFragment(hosts[i], diskIds != null ? diskIds[i] : -1, fragmentPair); } totalFragmentNum++; } @@ -285,21 +285,22 @@ public void removeFragment(FragmentPair fragmentPair) { diskIds = ((FileFragment)fragmentPair.getLeftFragment()).getDiskIds(); } for (int i = 0; i < hosts.length; i++) { + int diskId = diskIds == null ? -1 : diskIds[i]; String normalizedHost = NetUtils.normalizeHost(hosts[i]); Map diskFragmentMap = fragmentHostMapping.get(normalizedHost); if (diskFragmentMap != null) { - FragmentsPerDisk fragmentsPerDisk = diskFragmentMap.get(diskIds[i]); + FragmentsPerDisk fragmentsPerDisk = diskFragmentMap.get(diskId); if (fragmentsPerDisk != null) { boolean isRemoved = fragmentsPerDisk.removeFragmentPair(fragmentPair); if (isRemoved) { if (fragmentsPerDisk.size() == 0) { - diskFragmentMap.remove(diskIds[i]); + diskFragmentMap.remove(diskId); if (diskFragmentMap.size() == 0) { fragmentHostMapping.remove(normalizedHost); } } - HostAndDisk hostAndDisk = new HostAndDisk(normalizedHost, diskIds[i]); + HostAndDisk hostAndDisk = new HostAndDisk(normalizedHost, diskId); if (totalHostPriority.containsKey(hostAndDisk)) { PrioritizedHost prioritizedHost = totalHostPriority.get(hostAndDisk); updateHostPriority(prioritizedHost.hostAndDisk, prioritizedHost.priority-1); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java index 50a118e00b..26647295fd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java @@ -26,7 +26,9 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryIdFactory; import org.apache.tajo.QueryUnitAttemptId; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryUnitRequest; @@ -368,6 +370,7 @@ private void assignLeafTasks(List taskRequests) { long taskSize = adjustTaskSize(); LOG.info("Adjusted task size: " + taskSize); + TajoConf conf = subQuery.getContext().getConf(); // host local, disk local String normalized = NetUtils.normalizeHost(host); Integer diskId = hostDiskBalancerMap.get(normalized).getDiskId(container.containerID); @@ -378,13 +381,14 @@ private void assignLeafTasks(List taskRequests) { break; } - if (assignedFragmentSize + fragmentPair.getLeftFragment().getLength() > taskSize) { + if (assignedFragmentSize + + PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()) > taskSize) { break; } else { fragmentPairs.add(fragmentPair); - assignedFragmentSize += fragmentPair.getLeftFragment().getLength(); + assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()); if (fragmentPair.getRightFragment() != null) { - assignedFragmentSize += fragmentPair.getRightFragment().getLength(); + assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getRightFragment()); } } scheduledFragments.removeFragment(fragmentPair); @@ -400,13 +404,14 @@ private void assignLeafTasks(List taskRequests) { break; } - if (assignedFragmentSize + fragmentPair.getLeftFragment().getLength() > taskSize) { + if (assignedFragmentSize + + PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()) > taskSize) { break; } else { fragmentPairs.add(fragmentPair); - assignedFragmentSize += fragmentPair.getLeftFragment().getLength(); + assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()); if (fragmentPair.getRightFragment() != null) { - assignedFragmentSize += fragmentPair.getRightFragment().getLength(); + assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getRightFragment()); } } scheduledFragments.removeFragment(fragmentPair); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java index a769089f25..6859707618 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java @@ -245,7 +245,7 @@ private void addDataLocation(Fragment fragment) { diskIds = ((FileFragment)fragment).getDiskIds(); } for (int i = 0; i < hosts.length; i++) { - dataLocations.add(new DataLocation(hosts[i], diskIds[i])); + dataLocations.add(new DataLocation(hosts[i], diskIds == null ? -1 : diskIds[i])); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java b/tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java new file mode 100644 index 0000000000..b322a2ef30 --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.tajo.util.Bytes; + +import java.io.File; +import java.io.IOException; + +import static org.apache.hadoop.hbase.HConstants.REPLICATION_ENABLE_KEY; + +public class HBaseTestClusterUtil { + private Configuration conf; + private MiniHBaseCluster hbaseCluster; + private MiniZooKeeperCluster zkCluster; + private File testBaseDir; + public HBaseTestClusterUtil(Configuration conf, File testBaseDir) { + this.conf = conf; + this.testBaseDir = testBaseDir; + } + /** + * Returns the path to the default root dir the minicluster uses. + * Note: this does not cause the root dir to be created. + * @return Fully qualified path for the default hbase root dir + * @throws java.io.IOException + */ + public Path getDefaultRootDirPath() throws IOException { + FileSystem fs = FileSystem.get(this.conf); + return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase"); + } + + /** + * Creates an hbase rootdir in user home directory. Also creates hbase + * version file. Normally you won't make use of this method. Root hbasedir + * is created for you as part of mini cluster startup. You'd only use this + * method if you were doing manual operation. + * @return Fully qualified path to hbase root dir + * @throws java.io.IOException + */ + public Path createRootDir() throws IOException { + FileSystem fs = FileSystem.get(this.conf); + Path hbaseRootdir = getDefaultRootDirPath(); + FSUtils.setRootDir(this.conf, hbaseRootdir); + fs.mkdirs(hbaseRootdir); + FSUtils.setVersion(fs, hbaseRootdir); + return hbaseRootdir; + } + + public void stopHBaseCluster() throws Exception { + if (hbaseCluster != null) { + hbaseCluster.shutdown(); + } + + if (zkCluster != null) { + zkCluster.shutdown(); + } + } + + public void startHBaseCluster() throws Exception { + File zkDataPath = new File(testBaseDir, "zk"); + startMiniZKCluster(zkDataPath); + + System.setProperty("HBASE_ZNODE_FILE", testBaseDir + "/hbase_znode_file"); + if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + } + if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) { + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1); + } + conf.setBoolean(REPLICATION_ENABLE_KEY, false); + createRootDir(); + + Configuration c = HBaseConfiguration.create(this.conf); + + hbaseCluster = new MiniHBaseCluster(c, 1); + + // Don't leave here till we've done a successful scan of the hbase:meta + HTable t = new HTable(c, TableName.META_TABLE_NAME); + ResultScanner s = t.getScanner(new Scan()); + while (s.next() != null) { + continue; + } + s.close(); + t.close(); + } + + /** + * Start a mini ZK cluster. If the property "test.hbase.zookeeper.property.clientPort" is set + * the port mentionned is used as the default port for ZooKeeper. + */ + private MiniZooKeeperCluster startMiniZKCluster(final File dir) + throws Exception { + if (this.zkCluster != null) { + throw new IOException("Cluster already running at " + dir); + } + this.zkCluster = new MiniZooKeeperCluster(conf); + final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0); + if (defPort > 0){ + // If there is a port in the config file, we use it. + this.zkCluster.setDefaultClientPort(defPort); + } + int clientPort = this.zkCluster.startup(dir, 1); + this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); + return this.zkCluster; + } + + public Configuration getConf() { + return conf; + } + + public MiniZooKeeperCluster getMiniZooKeeperCluster() { + return zkCluster; + } + + public MiniHBaseCluster getMiniHBaseCluster() { + return hbaseCluster; + } + + public HTableDescriptor getTableDescriptor(String tableName) throws IOException { + HBaseAdmin admin = new HBaseAdmin(conf); + return admin.getTableDescriptor(Bytes.toBytes(tableName)); + } + + public void createTable(HTableDescriptor hTableDesc) throws IOException { + HBaseAdmin admin = new HBaseAdmin(conf); + admin.createTable(hTableDesc); + } +} diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index aec11f6d6f..4544959b00 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -64,6 +64,7 @@ public class TajoTestingCluster { private FileSystem defaultFS; private MiniDFSCluster dfsCluster; private MiniCatalogServer catalogServer; + private HBaseTestClusterUtil hbaseUtil; private TajoMaster tajoMaster; private List tajoWorkers = new ArrayList(); @@ -241,6 +242,10 @@ public FileSystem getDefaultFileSystem() { return this.defaultFS; } + public HBaseTestClusterUtil getHBaseUtil() { + return hbaseUtil; + } + //////////////////////////////////////////////////////// // Catalog Section //////////////////////////////////////////////////////// @@ -464,6 +469,9 @@ public void startMiniCluster(final int numSlaves, final String [] dataNodeHosts) startMiniDFSCluster(numDataNodes, this.clusterTestBuildDir, dataNodeHosts); this.dfsCluster.waitClusterUp(); + hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir); + hbaseUtil.startHBaseCluster(); + if(!standbyWorkerMode) { startMiniYarnCluster(); } @@ -561,6 +569,14 @@ public void shutdownMiniCluster() throws IOException { } } + if (this.hbaseUtil != null) { + try { + this.hbaseUtil.stopHBaseCluster(); + } catch (Exception e) { + System.err.println("error stopping hbase cluster: " + e); + } + } + if(this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) { if(!ShutdownHookManager.get().isShutdownInProgress()) { //TODO clean test dir when ShutdownInProgress diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 756dadc41f..b10b5829bc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -376,7 +376,7 @@ private void comparePath(List expectedFiles, FragmentProto[] fragments, int index = 0; for (int i = startIndex; i < startIndex + expectedSize; i++, index++) { - FileFragment fragment = FragmentConvertor.convert(util.getConfiguration(), StoreType.CSV, fragments[index]); + FileFragment fragment = FragmentConvertor.convert(util.getConfiguration(), fragments[index]); assertEquals(expectedFiles.get(i), fragment.getPath()); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 99c616d606..e8671bef54 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -199,8 +199,7 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St Preconditions.checkNotNull(ctx.getTable(scanNode.getTableName()), "Error: There is no table matched to %s", scanNode.getTableName()); - List fragments = FragmentConvertor.convert(ctx.getConf(), meta.getStoreType(), - ctx.getTables(scanNode.getTableName())); + List fragments = FragmentConvertor.convert(ctx.getConf(), ctx.getTables(scanNode.getTableName())); Datum[] datum = new Datum[]{DatumFactory.createInt4(rndKey)}; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java new file mode 100644 index 0000000000..baa04545b8 --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -0,0 +1,325 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.engine.eval.*; +import org.apache.tajo.engine.planner.PlannerUtil; +import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.hbase.HBaseFragment; +import org.apache.tajo.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.net.InetAddress; +import java.sql.ResultSet; +import java.text.DecimalFormat; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +@Category(IntegrationTest.class) +public class TestHBaseTable extends QueryTestCaseBase { + +// @Test +// public void testVerifyCreateHBaseTableRequiredMeta() throws Exception { +// try { +// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + +// "USING hbase").close(); +// +// fail("hbase table must have 'table' meta"); +// } catch (Exception e) { +// assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); +// } +// +// try { +// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + +// "USING hbase " + +// "WITH ('table'='hbase_table')").close(); +// +// fail("hbase table must have 'columns' meta"); +// } catch (Exception e) { +// assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); +// } +// +// try { +// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + +// "USING hbase " + +// "WITH ('table'='hbase_table', 'columns'='col1:,col2:')").close(); +// +// fail("hbase table must have 'hbase.zookeeper.quorum' meta"); +// } catch (Exception e) { +// assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); +// } +// } +// +// @Test +// public void testCreateHBaseTable() throws Exception { +// String hostName = InetAddress.getLocalHost().getHostName(); +// String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); +// assertNotNull(zkPort); +// +// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + +// "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " + +// "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + +// "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); +// +// assertTableExists("hbase_mapped_table1"); +// +// HTableDescriptor hTableDesc = testingCluster.getHBaseUtil().getTableDescriptor("hbase_table"); +// assertNotNull(hTableDesc); +// assertEquals("hbase_table", hTableDesc.getNameAsString()); +// +// HColumnDescriptor[] hColumns = hTableDesc.getColumnFamilies(); +// // col1 is mapped to rowkey +// assertEquals(2, hColumns.length); +// assertEquals("col2", hColumns[0].getNameAsString()); +// assertEquals("col3", hColumns[1].getNameAsString()); +// +// executeString("DROP TABLE hbase_mapped_table1 PURGE"); +// } +// +// @Test +// public void testCreateNotExistsExternalHBaseTable() throws Exception { +// String hostName = InetAddress.getLocalHost().getHostName(); +// String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); +// assertNotNull(zkPort); +// +// try { +// executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + +// "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " + +// "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + +// "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); +// fail("External table should be a existed table."); +// } catch (Exception e) { +// assertTrue(e.getMessage().indexOf("External table should be a existed table.") >= 0); +// } +// } +// +// @Test +// public void testCreateExternalHBaseTable() throws Exception { +// HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); +// hTableDesc.addFamily(new HColumnDescriptor("col1")); +// hTableDesc.addFamily(new HColumnDescriptor("col2")); +// hTableDesc.addFamily(new HColumnDescriptor("col3")); +// testingCluster.getHBaseUtil().createTable(hTableDesc); +// +// String hostName = InetAddress.getLocalHost().getHostName(); +// String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); +// assertNotNull(zkPort); +// +// executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + +// "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + +// "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + +// "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); +// +// assertTableExists("external_hbase_mapped_table"); +// +// executeString("DROP TABLE external_hbase_mapped_table PURGE"); +// } + + @Test + public void testSimpleSelectQuery() throws Exception { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col1")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("external_hbase_mapped_table"); + + + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(i).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk > '20'"); + assertResultSet(res); + cleanupQuery(res); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } + + @Test + public void testBinaryMappedQuery() throws Exception { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col1")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk int8, col1 text, col2 text, col3 int4)\n " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key#b,col1:a,col2:,col3:b#b', \n" + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "', \n" + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("external_hbase_mapped_table"); + + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + for (int i = 0; i < 100; i++) { + Put put = new Put(Bytes.toBytes((long) i)); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes(i)); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk > 20"); + assertResultSet(res); + res.close(); + + //Projection + res = executeString("select col3 from external_hbase_mapped_table where rk > 95"); + + String expected = "col3\n" + + "-------------------------------\n" + + "96\n" + + "97\n" + + "98\n" + + "99\n"; + + assertEquals(expected, resultSetToString(res)); + res.close(); + + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } + + @Test + public void testRowFieldSelectQuery() throws Exception { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, rk2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'='0:key,1:key,col3:a', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("external_hbase_mapped_table"); + + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + for (int i = 0; i < 100; i++) { + Put put = new Put(("field1-" + i + "_field2-" + i).getBytes()); + put.add("col3".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 > 'field1-20'"); + assertResultSet(res); + cleanupQuery(res); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } + + @Test + public void testIndexPredication() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + "'hbase.split.rowkeys'='010,040,060,080', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + + assertTableExists("external_hbase_mapped_table"); + HBaseAdmin hAdmin = new HBaseAdmin(HBaseConfiguration.create(conf)); + hAdmin.tableExists("external_hbase_table"); + + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + assertEquals(5, keys.getFirst().length); + + DecimalFormat df = new DecimalFormat("000"); + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(df.format(i)).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } + + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "external_hbase_mapped_table"); + + // where rk >= '020' and rk <= '055' + ScanNode scanNode = new ScanNode(1); + EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("020"))); + EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("055"))); + EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); + + scanNode.setQual(evalNodeA); + + StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE); + List fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, + PlannerUtil.getIndexPredications(storageManager, tableDesc, scanNode)); + + assertEquals(2, fragments.size()); + HBaseFragment fragment1 = (HBaseFragment)fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + HBaseFragment fragment2 = (HBaseFragment)fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("055", new String(fragment2.getStopRow())); + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk >= '020' and rk <= '055'"); + assertResultSet(res); + res.close(); + } +} diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testBinaryMappedQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testBinaryMappedQuery.result new file mode 100644 index 0000000000..8d50bf1b58 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testBinaryMappedQuery.result @@ -0,0 +1,81 @@ +rk,col1,col2,col3 +------------------------------- +21,a-21,{"k1":"k1-21", "k2":"k2-21"},21 +22,a-22,{"k1":"k1-22", "k2":"k2-22"},22 +23,a-23,{"k1":"k1-23", "k2":"k2-23"},23 +24,a-24,{"k1":"k1-24", "k2":"k2-24"},24 +25,a-25,{"k1":"k1-25", "k2":"k2-25"},25 +26,a-26,{"k1":"k1-26", "k2":"k2-26"},26 +27,a-27,{"k1":"k1-27", "k2":"k2-27"},27 +28,a-28,{"k1":"k1-28", "k2":"k2-28"},28 +29,a-29,{"k1":"k1-29", "k2":"k2-29"},29 +30,a-30,{"k1":"k1-30", "k2":"k2-30"},30 +31,a-31,{"k1":"k1-31", "k2":"k2-31"},31 +32,a-32,{"k1":"k1-32", "k2":"k2-32"},32 +33,a-33,{"k1":"k1-33", "k2":"k2-33"},33 +34,a-34,{"k1":"k1-34", "k2":"k2-34"},34 +35,a-35,{"k1":"k1-35", "k2":"k2-35"},35 +36,a-36,{"k1":"k1-36", "k2":"k2-36"},36 +37,a-37,{"k1":"k1-37", "k2":"k2-37"},37 +38,a-38,{"k1":"k1-38", "k2":"k2-38"},38 +39,a-39,{"k1":"k1-39", "k2":"k2-39"},39 +40,a-40,{"k1":"k1-40", "k2":"k2-40"},40 +41,a-41,{"k1":"k1-41", "k2":"k2-41"},41 +42,a-42,{"k1":"k1-42", "k2":"k2-42"},42 +43,a-43,{"k1":"k1-43", "k2":"k2-43"},43 +44,a-44,{"k1":"k1-44", "k2":"k2-44"},44 +45,a-45,{"k1":"k1-45", "k2":"k2-45"},45 +46,a-46,{"k1":"k1-46", "k2":"k2-46"},46 +47,a-47,{"k1":"k1-47", "k2":"k2-47"},47 +48,a-48,{"k1":"k1-48", "k2":"k2-48"},48 +49,a-49,{"k1":"k1-49", "k2":"k2-49"},49 +50,a-50,{"k1":"k1-50", "k2":"k2-50"},50 +51,a-51,{"k1":"k1-51", "k2":"k2-51"},51 +52,a-52,{"k1":"k1-52", "k2":"k2-52"},52 +53,a-53,{"k1":"k1-53", "k2":"k2-53"},53 +54,a-54,{"k1":"k1-54", "k2":"k2-54"},54 +55,a-55,{"k1":"k1-55", "k2":"k2-55"},55 +56,a-56,{"k1":"k1-56", "k2":"k2-56"},56 +57,a-57,{"k1":"k1-57", "k2":"k2-57"},57 +58,a-58,{"k1":"k1-58", "k2":"k2-58"},58 +59,a-59,{"k1":"k1-59", "k2":"k2-59"},59 +60,a-60,{"k1":"k1-60", "k2":"k2-60"},60 +61,a-61,{"k1":"k1-61", "k2":"k2-61"},61 +62,a-62,{"k1":"k1-62", "k2":"k2-62"},62 +63,a-63,{"k1":"k1-63", "k2":"k2-63"},63 +64,a-64,{"k1":"k1-64", "k2":"k2-64"},64 +65,a-65,{"k1":"k1-65", "k2":"k2-65"},65 +66,a-66,{"k1":"k1-66", "k2":"k2-66"},66 +67,a-67,{"k1":"k1-67", "k2":"k2-67"},67 +68,a-68,{"k1":"k1-68", "k2":"k2-68"},68 +69,a-69,{"k1":"k1-69", "k2":"k2-69"},69 +70,a-70,{"k1":"k1-70", "k2":"k2-70"},70 +71,a-71,{"k1":"k1-71", "k2":"k2-71"},71 +72,a-72,{"k1":"k1-72", "k2":"k2-72"},72 +73,a-73,{"k1":"k1-73", "k2":"k2-73"},73 +74,a-74,{"k1":"k1-74", "k2":"k2-74"},74 +75,a-75,{"k1":"k1-75", "k2":"k2-75"},75 +76,a-76,{"k1":"k1-76", "k2":"k2-76"},76 +77,a-77,{"k1":"k1-77", "k2":"k2-77"},77 +78,a-78,{"k1":"k1-78", "k2":"k2-78"},78 +79,a-79,{"k1":"k1-79", "k2":"k2-79"},79 +80,a-80,{"k1":"k1-80", "k2":"k2-80"},80 +81,a-81,{"k1":"k1-81", "k2":"k2-81"},81 +82,a-82,{"k1":"k1-82", "k2":"k2-82"},82 +83,a-83,{"k1":"k1-83", "k2":"k2-83"},83 +84,a-84,{"k1":"k1-84", "k2":"k2-84"},84 +85,a-85,{"k1":"k1-85", "k2":"k2-85"},85 +86,a-86,{"k1":"k1-86", "k2":"k2-86"},86 +87,a-87,{"k1":"k1-87", "k2":"k2-87"},87 +88,a-88,{"k1":"k1-88", "k2":"k2-88"},88 +89,a-89,{"k1":"k1-89", "k2":"k2-89"},89 +90,a-90,{"k1":"k1-90", "k2":"k2-90"},90 +91,a-91,{"k1":"k1-91", "k2":"k2-91"},91 +92,a-92,{"k1":"k1-92", "k2":"k2-92"},92 +93,a-93,{"k1":"k1-93", "k2":"k2-93"},93 +94,a-94,{"k1":"k1-94", "k2":"k2-94"},94 +95,a-95,{"k1":"k1-95", "k2":"k2-95"},95 +96,a-96,{"k1":"k1-96", "k2":"k2-96"},96 +97,a-97,{"k1":"k1-97", "k2":"k2-97"},97 +98,a-98,{"k1":"k1-98", "k2":"k2-98"},98 +99,a-99,{"k1":"k1-99", "k2":"k2-99"},99 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result b/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result new file mode 100644 index 0000000000..f38d238366 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result @@ -0,0 +1,38 @@ +rk,col1,col2,col3 +------------------------------- +020,a-20,{"k1":"k1-20", "k2":"k2-20"},b-20 +021,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21 +022,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22 +023,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23 +024,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24 +025,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25 +026,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26 +027,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27 +028,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28 +029,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29 +030,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30 +031,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31 +032,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32 +033,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33 +034,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34 +035,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35 +036,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36 +037,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37 +038,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38 +039,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39 +040,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40 +041,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41 +042,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42 +043,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43 +044,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44 +045,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45 +046,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46 +047,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47 +048,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48 +049,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49 +050,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50 +051,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51 +052,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52 +053,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53 +054,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54 +055,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result new file mode 100644 index 0000000000..34369df5a9 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result @@ -0,0 +1,88 @@ +rk1,rk2,col3 +------------------------------- +field1-21,field2-21,a-21 +field1-22,field2-22,a-22 +field1-23,field2-23,a-23 +field1-24,field2-24,a-24 +field1-25,field2-25,a-25 +field1-26,field2-26,a-26 +field1-27,field2-27,a-27 +field1-28,field2-28,a-28 +field1-29,field2-29,a-29 +field1-30,field2-30,a-30 +field1-31,field2-31,a-31 +field1-32,field2-32,a-32 +field1-33,field2-33,a-33 +field1-34,field2-34,a-34 +field1-35,field2-35,a-35 +field1-36,field2-36,a-36 +field1-37,field2-37,a-37 +field1-38,field2-38,a-38 +field1-39,field2-39,a-39 +field1-3,field2-3,a-3 +field1-40,field2-40,a-40 +field1-41,field2-41,a-41 +field1-42,field2-42,a-42 +field1-43,field2-43,a-43 +field1-44,field2-44,a-44 +field1-45,field2-45,a-45 +field1-46,field2-46,a-46 +field1-47,field2-47,a-47 +field1-48,field2-48,a-48 +field1-49,field2-49,a-49 +field1-4,field2-4,a-4 +field1-50,field2-50,a-50 +field1-51,field2-51,a-51 +field1-52,field2-52,a-52 +field1-53,field2-53,a-53 +field1-54,field2-54,a-54 +field1-55,field2-55,a-55 +field1-56,field2-56,a-56 +field1-57,field2-57,a-57 +field1-58,field2-58,a-58 +field1-59,field2-59,a-59 +field1-5,field2-5,a-5 +field1-60,field2-60,a-60 +field1-61,field2-61,a-61 +field1-62,field2-62,a-62 +field1-63,field2-63,a-63 +field1-64,field2-64,a-64 +field1-65,field2-65,a-65 +field1-66,field2-66,a-66 +field1-67,field2-67,a-67 +field1-68,field2-68,a-68 +field1-69,field2-69,a-69 +field1-6,field2-6,a-6 +field1-70,field2-70,a-70 +field1-71,field2-71,a-71 +field1-72,field2-72,a-72 +field1-73,field2-73,a-73 +field1-74,field2-74,a-74 +field1-75,field2-75,a-75 +field1-76,field2-76,a-76 +field1-77,field2-77,a-77 +field1-78,field2-78,a-78 +field1-79,field2-79,a-79 +field1-7,field2-7,a-7 +field1-80,field2-80,a-80 +field1-81,field2-81,a-81 +field1-82,field2-82,a-82 +field1-83,field2-83,a-83 +field1-84,field2-84,a-84 +field1-85,field2-85,a-85 +field1-86,field2-86,a-86 +field1-87,field2-87,a-87 +field1-88,field2-88,a-88 +field1-89,field2-89,a-89 +field1-8,field2-8,a-8 +field1-90,field2-90,a-90 +field1-91,field2-91,a-91 +field1-92,field2-92,a-92 +field1-93,field2-93,a-93 +field1-94,field2-94,a-94 +field1-95,field2-95,a-95 +field1-96,field2-96,a-96 +field1-97,field2-97,a-97 +field1-98,field2-98,a-98 +field1-99,field2-99,a-99 +field1-9,field2-9,a-9 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result new file mode 100644 index 0000000000..ef2f27a0d5 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result @@ -0,0 +1,88 @@ +rk,col1,col2,col3 +------------------------------- +21,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21 +22,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22 +23,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23 +24,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24 +25,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25 +26,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26 +27,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27 +28,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28 +29,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29 +3,a-3,{"k1":"k1-3", "k2":"k2-3"},b-3 +30,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30 +31,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31 +32,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32 +33,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33 +34,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34 +35,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35 +36,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36 +37,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37 +38,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38 +39,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39 +4,a-4,{"k1":"k1-4", "k2":"k2-4"},b-4 +40,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40 +41,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41 +42,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42 +43,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43 +44,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44 +45,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45 +46,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46 +47,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47 +48,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48 +49,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49 +5,a-5,{"k1":"k1-5", "k2":"k2-5"},b-5 +50,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50 +51,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51 +52,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52 +53,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53 +54,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54 +55,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55 +56,a-56,{"k1":"k1-56", "k2":"k2-56"},b-56 +57,a-57,{"k1":"k1-57", "k2":"k2-57"},b-57 +58,a-58,{"k1":"k1-58", "k2":"k2-58"},b-58 +59,a-59,{"k1":"k1-59", "k2":"k2-59"},b-59 +6,a-6,{"k1":"k1-6", "k2":"k2-6"},b-6 +60,a-60,{"k1":"k1-60", "k2":"k2-60"},b-60 +61,a-61,{"k1":"k1-61", "k2":"k2-61"},b-61 +62,a-62,{"k1":"k1-62", "k2":"k2-62"},b-62 +63,a-63,{"k1":"k1-63", "k2":"k2-63"},b-63 +64,a-64,{"k1":"k1-64", "k2":"k2-64"},b-64 +65,a-65,{"k1":"k1-65", "k2":"k2-65"},b-65 +66,a-66,{"k1":"k1-66", "k2":"k2-66"},b-66 +67,a-67,{"k1":"k1-67", "k2":"k2-67"},b-67 +68,a-68,{"k1":"k1-68", "k2":"k2-68"},b-68 +69,a-69,{"k1":"k1-69", "k2":"k2-69"},b-69 +7,a-7,{"k1":"k1-7", "k2":"k2-7"},b-7 +70,a-70,{"k1":"k1-70", "k2":"k2-70"},b-70 +71,a-71,{"k1":"k1-71", "k2":"k2-71"},b-71 +72,a-72,{"k1":"k1-72", "k2":"k2-72"},b-72 +73,a-73,{"k1":"k1-73", "k2":"k2-73"},b-73 +74,a-74,{"k1":"k1-74", "k2":"k2-74"},b-74 +75,a-75,{"k1":"k1-75", "k2":"k2-75"},b-75 +76,a-76,{"k1":"k1-76", "k2":"k2-76"},b-76 +77,a-77,{"k1":"k1-77", "k2":"k2-77"},b-77 +78,a-78,{"k1":"k1-78", "k2":"k2-78"},b-78 +79,a-79,{"k1":"k1-79", "k2":"k2-79"},b-79 +8,a-8,{"k1":"k1-8", "k2":"k2-8"},b-8 +80,a-80,{"k1":"k1-80", "k2":"k2-80"},b-80 +81,a-81,{"k1":"k1-81", "k2":"k2-81"},b-81 +82,a-82,{"k1":"k1-82", "k2":"k2-82"},b-82 +83,a-83,{"k1":"k1-83", "k2":"k2-83"},b-83 +84,a-84,{"k1":"k1-84", "k2":"k2-84"},b-84 +85,a-85,{"k1":"k1-85", "k2":"k2-85"},b-85 +86,a-86,{"k1":"k1-86", "k2":"k2-86"},b-86 +87,a-87,{"k1":"k1-87", "k2":"k2-87"},b-87 +88,a-88,{"k1":"k1-88", "k2":"k2-88"},b-88 +89,a-89,{"k1":"k1-89", "k2":"k2-89"},b-89 +9,a-9,{"k1":"k1-9", "k2":"k2-9"},b-9 +90,a-90,{"k1":"k1-90", "k2":"k2-90"},b-90 +91,a-91,{"k1":"k1-91", "k2":"k2-91"},b-91 +92,a-92,{"k1":"k1-92", "k2":"k2-92"},b-92 +93,a-93,{"k1":"k1-93", "k2":"k2-93"},b-93 +94,a-94,{"k1":"k1-94", "k2":"k2-94"},b-94 +95,a-95,{"k1":"k1-95", "k2":"k2-95"},b-95 +96,a-96,{"k1":"k1-96", "k2":"k2-96"},b-96 +97,a-97,{"k1":"k1-97", "k2":"k2-97"},b-97 +98,a-98,{"k1":"k1-98", "k2":"k2-98"},b-98 +99,a-99,{"k1":"k1-99", "k2":"k2-99"},b-99 \ No newline at end of file diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index c50fa66fa0..301d06ab56 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -34,6 +34,7 @@ UTF-8 1.5.0 2.1.0 + 0.98.6.1-hadoop2 @@ -131,6 +132,7 @@ --proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto --java_out=target/generated-sources/proto src/main/proto/IndexProtos.proto + src/main/proto/StorageFragmentProtos.proto @@ -314,6 +316,11 @@ parquet-format ${parquet.format.version} + + org.apache.hbase + hbase-client + ${hbase.version} + diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index c925cd8b79..d3b18cf672 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -27,10 +27,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.statistics.TableStats; @@ -639,7 +636,8 @@ public String getMessage(){ } @Override - public List getSplits(String tableName, TableDesc table) throws IOException { + public List getSplits(String tableName, TableDesc table, + List indexPredications) throws IOException { return getSplits(tableName, table.getMeta(), table.getSchema(), table.getPath()); } @@ -699,4 +697,8 @@ public void purgeTable(TableDesc tableDesc) throws IOException { } } + @Override + public Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { + return null; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/IndexPredication.java b/tajo-storage/src/main/java/org/apache/tajo/storage/IndexPredication.java new file mode 100644 index 0000000000..b3f9a487aa --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/IndexPredication.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; + +public class IndexPredication { + private Column column; + private int columnId; + private Datum startValue; + private Datum stopValue; + + public Column getColumn() { + return column; + } + + public void setColumn(Column column) { + this.column = column; + } + + public int getColumnId() { + return columnId; + } + + public void setColumnId(int columnId) { + this.columnId = columnId; + } + + public Datum getStartValue() { + return startValue; + } + + public void setStartValue(Datum startValue) { + this.startValue = startValue; + } + + public Datum getStopValue() { + return stopValue; + } + + public void setStopValue(Datum stopValue) { + this.stopValue = stopValue; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 23c2406804..b8a8034839 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; @@ -37,6 +38,7 @@ import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; +import org.apache.tajo.storage.hbase.HBaseStorageManager; import org.apache.tajo.util.Bytes; import org.apache.tajo.util.FileUtil; @@ -79,13 +81,19 @@ public abstract class StorageManager { protected abstract void storageInit() throws IOException ; public abstract void createTable(TableDesc tableDesc) throws IOException; public abstract void purgeTable(TableDesc tableDesc) throws IOException; - public abstract List getSplits(String fragmentId, TableDesc tableDesc) throws IOException; + public abstract List getSplits(String fragmentId, TableDesc tableDesc, + List indexPredications) throws IOException; + public abstract Column[] getIndexableColumns(TableDesc tableDesc) throws IOException; public void init(TajoConf tajoConf) throws IOException { this.conf = tajoConf; storageInit(); } + public List getSplits(String fragmentId, TableDesc tableDesc) throws IOException { + return getSplits(fragmentId, tableDesc, null); + } + public static FileStorageManager getFileStorageManager(TajoConf tajoConf) throws IOException { return getFileStorageManager(tajoConf, null); } @@ -112,6 +120,9 @@ public static synchronized StorageManager getStorageManager ( StorageManager manager = storageManagers.get(storeKey); if (manager == null) { switch (storeType) { + case HBASE: + manager = new HBaseStorageManager(); + break; default: manager = new FileStorageManager(); } @@ -125,7 +136,7 @@ public static synchronized StorageManager getStorageManager ( } public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException { - return getScanner(meta, schema, FragmentConvertor.convert(conf, meta.getStoreType(), fragment), target); + return getScanner(meta, schema, FragmentConvertor.convert(conf, fragment), target); } public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index dcd9f0a033..970d1428ea 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -24,6 +24,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.util.TUtil; import java.io.IOException; @@ -229,6 +230,7 @@ public FragmentProto getProto() { FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(this.tableName); + fragmentBuilder.setStoreType(StoreType.CSV.name()); fragmentBuilder.setContents(builder.buildPartial().toByteString()); return fragmentBuilder.build(); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java index 0315a8d4bd..0f5802da7a 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java @@ -49,16 +49,21 @@ public class FragmentConvertor { public static Class getFragmentClass(Configuration conf, StoreType storeType) throws IOException { - String handlerName = storeType.name().toLowerCase(); - Class fragmentClass = CACHED_FRAGMENT_CLASSES.get(handlerName); + String name = storeType.name().toLowerCase(); + return getFragmentClass(conf, name); + } + + public static Class getFragmentClass(Configuration conf, String storeType) + throws IOException { + Class fragmentClass = CACHED_FRAGMENT_CLASSES.get(storeType.toLowerCase()); if (fragmentClass == null) { fragmentClass = conf.getClass( - String.format("tajo.storage.fragment.%s.class", storeType.name().toLowerCase()), null, Fragment.class); - CACHED_FRAGMENT_CLASSES.put(handlerName, fragmentClass); + String.format("tajo.storage.fragment.%s.class", storeType.toLowerCase()), null, Fragment.class); + CACHED_FRAGMENT_CLASSES.put(storeType.toLowerCase(), fragmentClass); } if (fragmentClass == null) { - throw new IOException("No such a fragment for " + storeType.name()); + throw new IOException("No such a fragment for " + storeType.toLowerCase()); } return fragmentClass; @@ -81,11 +86,11 @@ public static T convert(Class clazz, FragmentProto fragm return result; } - public static T convert(Configuration conf, StoreType storeType, FragmentProto fragment) + public static T convert(Configuration conf, FragmentProto fragment) throws IOException { - Class fragmentClass = (Class) getFragmentClass(conf, storeType); + Class fragmentClass = (Class) getFragmentClass(conf, fragment.getStoreType().toLowerCase()); if (fragmentClass == null) { - throw new IOException("No such a fragment class for " + storeType.name()); + throw new IOException("No such a fragment class for " + fragment.getStoreType()); } return convert(fragmentClass, fragment); } @@ -102,14 +107,13 @@ public static List convert(Class clazz, FragmentProto return list; } - public static List convert(Configuration conf, StoreType storeType, - FragmentProto...fragments) throws IOException { + public static List convert(Configuration conf, FragmentProto...fragments) throws IOException { List list = Lists.newArrayList(); if (fragments == null) { return list; } for (FragmentProto proto : fragments) { - list.add((T) convert(conf, storeType, proto)); + list.add((T) convert(conf, proto)); } return list; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java new file mode 100644 index 0000000000..0c9b20edde --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.util.Pair; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ColumnMapping { + private TableMeta tableMeta; + private Schema schema; + private char rowKeyDelimiter; + + private int[] rowKeyFieldIndexes; + private boolean[] isRowKeyMappings; + private boolean[] isBinaryColumns; + + // schema order -> 0: cf name, 1: column name -> name bytes + private byte[][][] mappingColumns; + + public ColumnMapping(Schema schema, TableMeta tableMeta) throws IOException { + this.schema = schema; + this.tableMeta = tableMeta; + + init(); + } + + public void init() throws IOException { + String delim = tableMeta.getOption(HBaseStorageManager.META_ROWKEY_DELIMITER, "").trim(); + if (delim.length() > 0) { + rowKeyDelimiter = delim.charAt(0); + } + isRowKeyMappings = new boolean[schema.size()]; + rowKeyFieldIndexes = new int[schema.size()]; + isBinaryColumns = new boolean[schema.size()]; + + mappingColumns = new byte[schema.size()][][]; + + for (int i = 0; i < schema.size(); i++) { + isRowKeyMappings[i] = false; + rowKeyFieldIndexes[i] = -1; + isBinaryColumns[i] = false; + } + + List> hbaseColumnMappings = parseColumnMapping(tableMeta); + if (hbaseColumnMappings == null || hbaseColumnMappings.isEmpty()) { + throw new IOException("columns property is required."); + } + + if (hbaseColumnMappings.size() != schema.getColumns().size()) { + throw new IOException("The number of mapped HBase columns is great than the number of Tajo table columns"); + } + + int index = 0; + for (Pair eachMapping: hbaseColumnMappings) { + String cfName = eachMapping.getFirst(); + String columnName = eachMapping.getSecond(); + + mappingColumns[index] = new byte[2][]; + + RowKeyMapping rowKeyMapping = HBaseStorageManager.getRowKeyMapping(cfName, columnName); + if (rowKeyMapping != null) { + isRowKeyMappings[index] = true; + isBinaryColumns[index] = rowKeyMapping.isBinary(); + if (!cfName.isEmpty()) { + if (rowKeyDelimiter == 0) { + throw new IOException("hbase.rowkey.delimiter is required."); + } + rowKeyFieldIndexes[index] = Integer.parseInt(cfName); + } else { + rowKeyFieldIndexes[index] = -1; //rowkey is mapped a single column. + } + } else { + isRowKeyMappings[index] = false; + + if (cfName != null) { + mappingColumns[index][0] = Bytes.toBytes(cfName); + } + + if (columnName != null) { + String[] columnNameTokens = columnName.split("#"); + if (columnNameTokens[0].isEmpty()) { + mappingColumns[index][1] = null; + } else { + mappingColumns[index][1] = Bytes.toBytes(columnNameTokens[0]); + } + if (columnNameTokens.length == 2 && "b".equals(columnNameTokens[1])) { + isBinaryColumns[index] = true; + } + } + } + + index++; + } + } + + public char getRowKeyDelimiter() { + return rowKeyDelimiter; + } + + public int[] getRowKeyFieldIndexes() { + return rowKeyFieldIndexes; + } + + public boolean[] getIsRowKeyMappings() { + return isRowKeyMappings; + } + + public byte[][][] getMappingColumns() { + return mappingColumns; + } + + public Schema getSchema() { + return schema; + } + + public boolean[] getIsBinaryColumns() { + return isBinaryColumns; + } + + /** + * Get column mapping data from tableMeta's option. + * First value of return is column family name and second value is column name which can be null. + * @param tableMeta + * @return + * @throws java.io.IOException + */ + public static List> parseColumnMapping(TableMeta tableMeta) throws IOException { + List> columnMappings = new ArrayList>(); + + String columnMapping = tableMeta.getOption(HBaseStorageManager.META_COLUMNS_KEY, ""); + if (columnMapping == null || columnMapping.trim().isEmpty()) { + return columnMappings; + } + + for (String eachToken: columnMapping.split(",")) { + String[] cfToken = eachToken.split(":"); + + String cfName = cfToken[0]; + String columnName = null; + if (cfToken.length == 2 && !cfToken[1].trim().isEmpty()) { + columnName = cfToken[1].trim(); + } + Pair mappingEntry = new Pair(cfName, columnName); + columnMappings.add(mappingEntry); + } + + return columnMappings; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java new file mode 100644 index 0000000000..54f69dfe78 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.Tuple; + +import java.io.IOException; + +public class HBaseAppender implements Appender { + @Override + public void init() throws IOException { + + } + + @Override + public void addTuple(Tuple t) throws IOException { + + } + + @Override + public void flush() throws IOException { + + } + + @Override + public long getEstimatedOutputSize() throws IOException { + return 0; + } + + @Override + public void close() throws IOException { + + } + + @Override + public void enableStats() { + + } + + @Override + public TableStats getStats() { + return null; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java new file mode 100644 index 0000000000..9984a7caf7 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.storage.SerializerDeserializer; +import org.apache.tajo.util.Bytes; + +import java.io.IOException; +import java.io.OutputStream; + +public class HBaseBinarySerializerDeserializer implements SerializerDeserializer { + @Override + public int serialize(Column col, Datum datum, OutputStream out, byte[] nullCharacters) throws IOException { + return 0; + } + + @Override + public Datum deserialize(Column col, byte[] bytes, int offset, int length, byte[] nullCharacters) throws IOException { + Datum datum; + switch (col.getDataType().getType()) { + case INT1: + case INT2: + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt2(Bytes.toShort(bytes, offset, length)); + break; + case INT4: + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes, offset, length)); + break; + case INT8: + if (length == 4) { + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes, offset, length)); + } else { + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes, offset, length)); + } + break; + case FLOAT4: + datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat4(Bytes.toFloat(bytes, offset)); + break; + case FLOAT8: + datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat8(Bytes.toDouble(bytes, offset)); + break; + case TEXT: + datum = bytes == null ? NullDatum.get() : DatumFactory.createText(bytes); + break; + default: + datum = NullDatum.get(); + break; + } + return datum; + } + + public static byte[] serialize(Column col, Datum datum) throws IOException { + if (datum == null || datum instanceof NullDatum) { + return null; + } + + byte[] bytes; + switch (col.getDataType().getType()) { + case INT1: + case INT2: + bytes = Bytes.toBytes(datum.asInt2()); + break; + case INT4: + bytes = Bytes.toBytes(datum.asInt4()); + break; + case INT8: + bytes = Bytes.toBytes(datum.asInt8()); + break; + case FLOAT4: + bytes = Bytes.toBytes(datum.asFloat4()); + break; + case FLOAT8: + bytes = Bytes.toBytes(datum.asFloat8()); + break; + case TEXT: + bytes = Bytes.toBytes(datum.asChars()); + break; + default: + bytes = null; + break; + } + + return bytes; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java new file mode 100644 index 0000000000..225916cc23 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.fragment.StorageFragmentProtos.HBaseFragmentProto; + +public class HBaseFragment implements Fragment, Comparable, Cloneable { + @Expose + private String tableName; + @Expose + private String hbaseTableName; + @Expose + private byte[] startRow; + @Expose + private byte[] stopRow; + @Expose + private String regionLocation; + @Expose + private boolean last; + @Expose + private long length; + + public HBaseFragment(String tableName, String hbaseTableName, byte[] startRow, byte[] stopRow, String regionLocation) { + this.tableName = tableName; + this.hbaseTableName = hbaseTableName; + this.startRow = startRow; + this.stopRow = stopRow; + this.regionLocation = regionLocation; + this.last = false; + } + + public HBaseFragment(ByteString raw) throws InvalidProtocolBufferException { + HBaseFragmentProto.Builder builder = HBaseFragmentProto.newBuilder(); + builder.mergeFrom(raw); + builder.build(); + init(builder.build()); + } + + private void init(HBaseFragmentProto proto) { + this.tableName = proto.getTableName(); + this.hbaseTableName = proto.getHbaseTableName(); + this.startRow = proto.getStartRow().toByteArray(); + this.stopRow = proto.getStopRow().toByteArray(); + this.regionLocation = proto.getRegionLocation(); + this.last = proto.getLast(); + } + + @Override + public int compareTo(HBaseFragment t) { + return Bytes.compareTo(startRow, t.startRow); + } + + @Override + public String getTableName() { + return tableName; + } + + @Override + public String getKey() { + return new String(startRow); + } + + @Override + public boolean isEmpty() { + return startRow == null || stopRow == null; + } + + @Override + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } + + @Override + public String[] getHosts() { + return new String[] {regionLocation}; + } + + public Object clone() throws CloneNotSupportedException { + HBaseFragment frag = (HBaseFragment) super.clone(); + frag.tableName = tableName; + frag.hbaseTableName = hbaseTableName; + frag.startRow = startRow; + frag.stopRow = stopRow; + frag.regionLocation = regionLocation; + frag.last = last; + frag.length = length; + return frag; + } + + @Override + public boolean equals(Object o) { + if (o instanceof HBaseFragment) { + HBaseFragment t = (HBaseFragment) o; + if (tableName.equals(t.tableName) + && Bytes.equals(startRow, t.startRow) + && Bytes.equals(stopRow, t.stopRow)) { + return true; + } + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(tableName, hbaseTableName, startRow, stopRow); + } + + @Override + public String toString() { + return "\"fragment\": {\"tableName\": \""+ tableName + "\", hbaseTableName\": \"" + hbaseTableName + "\"" + + ", \"startRow\": \"" + new String(startRow) + "\"" + + ", \"stopRow\": \"" + new String(stopRow) + "\"" + + ", \"length\": \"" + length + "\"}" ; + } + + @Override + public FragmentProto getProto() { + HBaseFragmentProto.Builder builder = HBaseFragmentProto.newBuilder(); + builder.setTableName(tableName) + .setHbaseTableName(hbaseTableName) + .setStartRow(ByteString.copyFrom(startRow)) + .setStopRow(ByteString.copyFrom(stopRow)) + .setLast(last) + .setLength(length) + .setRegionLocation(regionLocation); + + FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); + fragmentBuilder.setId(this.tableName); + fragmentBuilder.setContents(builder.buildPartial().toByteString()); + fragmentBuilder.setStoreType(StoreType.HBASE.name()); + return fragmentBuilder.build(); + } + + public byte[] getStartRow() { + return startRow; + } + + public byte[] getStopRow() { + return stopRow; + } + + public String getRegionLocation() { + return regionLocation; + } + + public boolean isLast() { + return last; + } + + public void setLast(boolean last) { + this.last = last; + } + + public String getHbaseTableName() { + return hbaseTableName; + } + + public void setHbaseTableName(String hbaseTableName) { + this.hbaseTableName = hbaseTableName; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java new file mode 100644 index 0000000000..8e423fdcb1 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java @@ -0,0 +1,318 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.client.Result; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.datum.ProtobufDatum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.BytesUtils; + +import java.util.Arrays; +import java.util.NavigableMap; + +public class HBaseLazyTuple implements Tuple, Cloneable { + private static final Log LOG = LogFactory.getLog(HBaseLazyTuple.class); + + private Datum[] values; + private Result result; + private byte[][][] mappingColumnFamilies; + private boolean[] isRowKeyMappings; + private boolean[] isBinaryColumns; + private int[] rowKeyFieldIndexes; + private char rowKeyDelimiter; + private HBaseTextSerializerDeserializer textSerde; + private HBaseBinarySerializerDeserializer binarySerde; + private Column[] schemaColumns; + + public HBaseLazyTuple(ColumnMapping columnMapping, + Column[] schemaColumns, + int[] targetIndexes, + HBaseTextSerializerDeserializer textSerde, + HBaseBinarySerializerDeserializer binarySerde, + Result result) { + values = new Datum[schemaColumns.length]; + mappingColumnFamilies = columnMapping.getMappingColumns(); + isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + isBinaryColumns = columnMapping.getIsBinaryColumns(); + rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); + rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); + + this.result = result; + this.schemaColumns = schemaColumns; + this.textSerde = textSerde; + this.binarySerde = binarySerde; + } + + @Override + public int size() { + return values.length; + } + + @Override + public boolean contains(int fieldid) { + return false; + } + + @Override + public boolean isNull(int fieldid) { + return false; + } + + @Override + public void clear() { + values = new Datum[schemaColumns.length]; + } + + @Override + public void put(int fieldId, Datum value) { + values[fieldId] = value; + } + + @Override + public void put(int fieldId, Datum[] values) { + for (int i = fieldId, j = 0; j < values.length; i++, j++) { + this.values[i] = values[j]; + } + } + + @Override + public void put(int fieldId, Tuple tuple) { + for (int i = fieldId, j = 0; j < tuple.size(); i++, j++) { + values[i] = tuple.get(j); + } + } + + @Override + public void put(Datum[] values) { + System.arraycopy(values, 0, this.values, 0, size()); + } + + @Override + public Datum get(int fieldId) { + if (values[fieldId] != null) { + return values[fieldId]; + } + + byte[] value = null; + if (isRowKeyMappings[fieldId]) { + value = result.getRow(); + if (!isBinaryColumns[fieldId] && rowKeyFieldIndexes[fieldId] >= 0) { + int rowKeyFieldIndex = rowKeyFieldIndexes[fieldId]; + + byte[][] rowKeyFields = BytesUtils.splitPreserveAllTokens(value, rowKeyDelimiter); + + if (rowKeyFields.length < rowKeyFieldIndex) { + values[fieldId] = NullDatum.get(); + return values[fieldId]; + } else { + value = rowKeyFields[rowKeyFieldIndex]; + } + } + } else { + if (mappingColumnFamilies[fieldId][1] == null) { + NavigableMap cfMap = result.getFamilyMap(mappingColumnFamilies[fieldId][0]); + if (cfMap != null && !cfMap.isEmpty()) { + int count = 0; + String delim = ""; + + StringBuilder sb = new StringBuilder(); + sb.append("{"); + for (NavigableMap.Entry entry: cfMap.entrySet()) { + byte[] entryKey = entry.getKey(); + byte[] entryValue = entry.getValue(); + + String keyText = new String(entryKey); + String valueText = null; + if (entryValue != null) { + try { + if (isBinaryColumns[fieldId]) { + valueText = binarySerde.deserialize( + schemaColumns[fieldId], entryValue, 0, entryValue.length, null).asChars(); + } else { + valueText = textSerde.deserialize( + schemaColumns[fieldId], entryValue, 0, entryValue.length, null).asChars(); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + sb.append(delim).append("\"").append(keyText).append("\":\"").append(valueText).append("\""); + delim = ", "; + if (count > 100) { + break; + } + } + sb.append("}"); + values[fieldId] = new TextDatum(sb.toString()); + return values[fieldId]; + } else { + value = null; + } + } else { + value = result.getValue(mappingColumnFamilies[fieldId][0], mappingColumnFamilies[fieldId][1]); + } + } + + if (value == null) { + values[fieldId] = NullDatum.get(); + } else { + try { + if (isBinaryColumns[fieldId]) { + values[fieldId] = binarySerde.deserialize(schemaColumns[fieldId], value, 0, value.length, null); + } else { + values[fieldId] = textSerde.deserialize(schemaColumns[fieldId], value, 0, value.length, null); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + + return values[fieldId]; + } + + @Override + public void setOffset(long offset) { + } + + @Override + public long getOffset() { + return 0; + } + + @Override + public boolean getBool(int fieldId) { + return get(fieldId).asBool(); + } + + @Override + public byte getByte(int fieldId) { + return get(fieldId).asByte(); + } + + @Override + public char getChar(int fieldId) { + return get(fieldId).asChar(); + } + + @Override + public byte [] getBytes(int fieldId) { + return get(fieldId).asByteArray(); + } + + @Override + public short getInt2(int fieldId) { + return get(fieldId).asInt2(); + } + + @Override + public int getInt4(int fieldId) { + return get(fieldId).asInt4(); + } + + @Override + public long getInt8(int fieldId) { + return get(fieldId).asInt8(); + } + + @Override + public float getFloat4(int fieldId) { + return get(fieldId).asFloat4(); + } + + @Override + public double getFloat8(int fieldId) { + return get(fieldId).asFloat8(); + } + + @Override + public String getText(int fieldId) { + return get(fieldId).asChars(); + } + + @Override + public ProtobufDatum getProtobufDatum(int fieldId) { + throw new UnsupportedException(); + } + + @Override + public char[] getUnicodeChars(int fieldId) { + return get(fieldId).asUnicodeChars(); + } + + public String toString() { + boolean first = true; + StringBuilder str = new StringBuilder(); + str.append("("); + Datum d; + for (int i = 0; i < values.length; i++) { + d = get(i); + if (d != null) { + if (first) { + first = false; + } else { + str.append(", "); + } + str.append(i) + .append("=>") + .append(d); + } + } + str.append(")"); + return str.toString(); + } + + @Override + public int hashCode() { + return Arrays.hashCode(values); + } + + @Override + public Datum[] getValues() { + Datum[] datums = new Datum[values.length]; + for (int i = 0; i < values.length; i++) { + datums[i] = get(i); + } + return datums; + } + + @Override + public Tuple clone() throws CloneNotSupportedException { + HBaseLazyTuple lazyTuple = (HBaseLazyTuple) super.clone(); + lazyTuple.values = getValues(); //shallow copy + return lazyTuple; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Tuple) { + Tuple other = (Tuple) obj; + return Arrays.equals(getValues(), other.getValues()); + } + return false; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java new file mode 100644 index 0000000000..87faf3b917 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.ColumnStats; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.Scanner; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.util.Bytes; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +public class HBaseScanner implements Scanner { + private static final Log LOG = LogFactory.getLog(HBaseScanner.class); + private static final int DEFAULT_FETCH_SZIE = 1000; + + protected boolean inited = false; + private TajoConf conf; + private Schema schema; + private TableMeta meta; + private HBaseFragment fragment; + private Scan scan; + private HTable htable; + private Configuration hbaseConf; + private Column[] targets; + private TableStats tableStats; + private ResultScanner scanner; + private AtomicBoolean finished = new AtomicBoolean(false); + private float progress = 0.0f; + private int scanFetchSize; + private Result[] scanResults; + private int scanResultIndex = -1; + private Column[] schemaColumns; + + private ColumnMapping columnMapping; + private int[] targetIndexes; + + private HBaseTextSerializerDeserializer textSerde; + private HBaseBinarySerializerDeserializer binarySerde; + + private int numRows = 0; + + public HBaseScanner (Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { + this.conf = (TajoConf)conf; + this.schema = schema; + this.meta = meta; + this.fragment = (HBaseFragment)fragment; + this.tableStats = new TableStats(); + } + + @Override + public void init() throws IOException { + inited = true; + schemaColumns = schema.toArray(); + if (fragment != null) { + tableStats.setNumBytes(0); + tableStats.setNumBlocks(1); + } + if (schema != null) { + for(Column eachColumn: schema.getColumns()) { + ColumnStats columnStats = new ColumnStats(eachColumn); + tableStats.addColumnStat(columnStats); + } + } + + textSerde = new HBaseTextSerializerDeserializer(); + binarySerde = new HBaseBinarySerializerDeserializer(); + + scanFetchSize = Integer.parseInt(meta.getOption("hbase.scanner.fetch,size", "" + DEFAULT_FETCH_SZIE)); + if (targets == null) { + targets = schema.toArray(); + } + + columnMapping = new ColumnMapping(schema, meta); + targetIndexes = new int[targets.length]; + int index = 0; + for (Column eachTargetColumn: targets) { + targetIndexes[index++] = schema.getColumnId(eachTargetColumn.getQualifiedName()); + } + + hbaseConf = HBaseStorageManager.getHBaseConfiguration(conf, meta); + + initScanner(); + } + + private void initScanner() throws IOException { + scan = new Scan(); + scan.setBatch(scanFetchSize); + scan.setStartRow(fragment.getStartRow()); + if (fragment.isLast() && fragment.getStopRow() != null && + fragment.getStopRow().length > 0) { + // last and stopRow is not empty + Filter filter = new InclusiveStopFilter(fragment.getStopRow()); + scan.setFilter(filter); + } else { + scan.setStopRow(fragment.getStopRow()); + } + + boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + for (int eachIndex: targetIndexes) { + if (isRowKeyMappings[eachIndex]) { + continue; + } + byte[][] mappingColumn = columnMapping.getMappingColumns()[eachIndex]; + if (mappingColumn[1] == null) { + scan.addFamily(mappingColumn[0]); + } else { + scan.addColumn(mappingColumn[0], mappingColumn[1]); + } + } + + htable = new HTable(hbaseConf, fragment.getHbaseTableName()); + scanner = htable.getScanner(scan); + } + + @Override + public Tuple next() throws IOException { + if (finished.get()) { + return null; + } + + if (scanResults == null || scanResultIndex >= scanResults.length) { + scanResults = scanner.next(scanFetchSize); + if (scanResults == null || scanResults.length == 0) { + finished.set(true); + progress = 1.0f; + return null; + } + scanResultIndex = 0; + } + + Result result = scanResults[scanResultIndex++]; + numRows++; + return new HBaseLazyTuple(columnMapping, schemaColumns, targetIndexes, textSerde, binarySerde, result); + } + + @Override + public void reset() throws IOException { + progress = 0.0f; + scanResultIndex = -1; + scanResults = null; + finished.set(false); + tableStats = new TableStats(); + + if (scanner != null) { + scanner.close(); + } + + initScanner(); + } + + @Override + public void close() throws IOException { + progress = 1.0f; + finished.set(true); + if (scanner != null) { + try { + scanner.close(); + } catch (Exception e) { + LOG.warn("Error while closing hbase scanner: " + e.getMessage(), e); + } + } + if (htable != null) { + htable.close(); + } + } + + @Override + public boolean isProjectable() { + return true; + } + + @Override + public void setTarget(Column[] targets) { + if (inited) { + throw new IllegalStateException("Should be called before init()"); + } + this.targets = targets; + } + + @Override + public boolean isSelectable() { + return false; + } + + @Override + public void setSearchCondition(Object expr) { + // TODO implements adding column filter to scanner. + } + + @Override + public boolean isSplittable() { + return true; + } + + @Override + public float getProgress() { + return progress; + } + + @Override + public TableStats getInputStats() { + tableStats.setNumRows(numRows); + return tableStats; + } + + @Override + public Schema getSchema() { + return schema; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java new file mode 100644 index 0000000000..01f394cc04 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -0,0 +1,399 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.storage.IndexPredication; +import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.util.Bytes; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.*; + +public class HBaseStorageManager extends StorageManager { + private final Log LOG = LogFactory.getLog(HBaseStorageManager.class); + + public static final String META_TABLE_KEY = "table"; + public static final String META_COLUMNS_KEY = "columns"; + public static final String META_SPLIT_ROW_KEYS_KEY = "hbase.split.rowkeys"; + public static final String META_SPLIT_ROW_KEYS_FILE_KEY = "hbase.split.rowkeys.file"; + public static final String META_ZK_QUORUM_KEY = "hbase.zookeeper.quorum"; + public static final String ROWKEY_COLUMN_MAPPING = "key"; + public static final String META_ROWKEY_DELIMITER = "hbase.rowkey.delimiter"; + + @Override + public void storageInit() throws IOException { + } + + @Override + public void createTable(TableDesc tableDesc) throws IOException { + TableMeta tableMeta = tableDesc.getMeta(); + + String hbaseTableName = tableMeta.getOption(META_TABLE_KEY, ""); + if (hbaseTableName == null || hbaseTableName.trim().isEmpty()) { + throw new IOException("HBase mapped table is required a '" + META_TABLE_KEY + "' attribute."); + } + TableName hTableName = TableName.valueOf(hbaseTableName); + + String columnMapping = tableMeta.getOption(META_COLUMNS_KEY, ""); + if (columnMapping != null && columnMapping.split(",").length > tableDesc.getSchema().size()) { + throw new IOException("Columns property has more entry than Tajo table columns"); + } + HBaseAdmin hAdmin = new HBaseAdmin(getHBaseConfiguration(conf, tableMeta)); + + if (tableDesc.isExternal()) { + // If tajo table is external table, only check validation. + if (columnMapping == null || columnMapping.isEmpty()) { + throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); + } + if (!hAdmin.tableExists(hTableName)) { + throw new IOException ("HBase table [" + hbaseTableName + "] not exists. " + + "External table should be a existed table."); + } + HTableDescriptor hTableDescriptor = hAdmin.getTableDescriptor(hTableName); + Set tableColumnFamilies = new HashSet(); + for (HColumnDescriptor eachColumn: hTableDescriptor.getColumnFamilies()) { + tableColumnFamilies.add(eachColumn.getNameAsString()); + } + + Collection mappingColumnFamilies = getColumnFamilies(columnMapping); + if (mappingColumnFamilies.isEmpty()) { + throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); + } + + for (String eachMappingColumnFamily: mappingColumnFamilies) { + if (!tableColumnFamilies.contains(eachMappingColumnFamily)) { + throw new IOException ("There is no " + eachMappingColumnFamily + " column family in " + hbaseTableName); + } + } + } else { + if (hAdmin.tableExists(hbaseTableName)) { + throw new IOException ("HBase table [" + hbaseTableName + "] already exists."); + } + // Creating hbase table + HTableDescriptor hTableDescriptor = parseHTableDescriptor(tableDesc); + + byte[][] splitKeys = getSplitKeys(conf, tableMeta); + if (splitKeys == null) { + hAdmin.createTable(hTableDescriptor); + } else { + hAdmin.createTable(hTableDescriptor, splitKeys); + } + } + + TableStats stats = new TableStats(); + stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); + tableDesc.setStats(stats); + } + + private byte[][] getSplitKeys(TajoConf conf, TableMeta meta) throws IOException { + String splitRowKeys = meta.getOption(META_SPLIT_ROW_KEYS_KEY, ""); + String splitRowKeysFile = meta.getOption(META_SPLIT_ROW_KEYS_FILE_KEY, ""); + + if ((splitRowKeys == null || splitRowKeys.isEmpty()) && + (splitRowKeysFile == null || splitRowKeysFile.isEmpty())) { + return null; + } + + if (splitRowKeys != null && !splitRowKeys.isEmpty()) { + String[] splitKeyTokens = splitRowKeys.split(","); + byte[][] splitKeys = new byte[splitKeyTokens.length][]; + for (int i = 0; i < splitKeyTokens.length; i++) { + splitKeys[i] = Bytes.toBytes(splitKeyTokens[i]); + } + return splitKeys; + } + + if (splitRowKeysFile != null && !splitRowKeysFile.isEmpty()) { + Path path = new Path(splitRowKeysFile); + FileSystem fs = path.getFileSystem(conf); + if (!fs.exists(path)) { + throw new IOException("hbase.split.rowkeys.file=" + path.toString() + " not exists."); + } + + SortedSet splitKeySet = new TreeSet(); + BufferedReader reader = null; + try { + reader = new BufferedReader(new InputStreamReader(fs.open(path))); + String line = null; + while ( (line = reader.readLine()) != null ) { + if (line.isEmpty()) { + continue; + } + splitKeySet.add(line); + } + } finally { + if (reader != null) { + reader.close(); + } + } + + if (splitKeySet.isEmpty()) { + return null; + } + + byte[][] splitKeys = new byte[splitKeySet.size()][]; + int index = 0; + for (String eachKey: splitKeySet) { + splitKeys[index++] = Bytes.toBytes(eachKey); + } + + return splitKeys; + } + + return null; + } + + private static List getColumnFamilies(String columnMapping) { + // columnMapping can have a duplicated column name as CF1:a, CF1:b + List columnFamilies = new ArrayList(); + + if (columnMapping == null) { + return columnFamilies; + } + + for (String eachToken: columnMapping.split(",")) { + String[] cfTokens = eachToken.trim().split(":"); + if (cfTokens.length == 2 && cfTokens[1] != null && getRowKeyMapping(cfTokens[0], cfTokens[1].trim()) != null) { + // rowkey + continue; + } + if (!columnFamilies.contains(cfTokens[0])) { + String[] binaryTokens = cfTokens[0].split("#"); + columnFamilies.add(binaryTokens[0]); + } + } + + return columnFamilies; + } + + public static Configuration getHBaseConfiguration(TajoConf tajoConf, TableMeta tableMeta) throws IOException { + String zkQuorum = tableMeta.getOption(META_ZK_QUORUM_KEY, ""); + if (zkQuorum == null || zkQuorum.trim().isEmpty()) { + throw new IOException("HBase mapped table is required a '" + META_ZK_QUORUM_KEY + "' attribute."); + } + + Configuration hbaseConf = (tajoConf == null) ? HBaseConfiguration.create() : HBaseConfiguration.create(tajoConf); + hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum); + + for (Map.Entry eachOption: tableMeta.getOptions().getAllKeyValus().entrySet()) { + String key = eachOption.getKey(); + if (key.startsWith(HConstants.ZK_CFG_PROPERTY_PREFIX)) { + hbaseConf.set(key, eachOption.getValue()); + } + } + return hbaseConf; + } + + public static HTableDescriptor parseHTableDescriptor(TableDesc tableDesc) throws IOException { + TableMeta tableMeta = tableDesc.getMeta(); + + String hbaseTableName = tableMeta.getOption(META_TABLE_KEY, ""); + if (hbaseTableName == null || hbaseTableName.trim().isEmpty()) { + throw new IOException("HBase mapped table is required a '" + META_TABLE_KEY + "' attribute."); + } + TableName hTableName = TableName.valueOf(hbaseTableName); + + String columnMapping = tableMeta.getOption(META_COLUMNS_KEY, ""); + if (columnMapping != null && columnMapping.split(",").length > tableDesc.getSchema().size()) { + throw new IOException("Columns property has more entry than Tajo table columns"); + } + HTableDescriptor hTableDescriptor = new HTableDescriptor(hTableName); + + Collection columnFamilies = getColumnFamilies(columnMapping); + //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. + if (columnFamilies.isEmpty()) { + for (Column eachColumn: tableDesc.getSchema().getColumns()) { + columnFamilies.add(eachColumn.getSimpleName()); + } + } + + for (String eachColumnFamily: columnFamilies) { + hTableDescriptor.addFamily(new HColumnDescriptor(eachColumnFamily)); + } + + return hTableDescriptor; + } + + @Override + public void purgeTable(TableDesc tableDesc) throws IOException { + HBaseAdmin hAdmin = new HBaseAdmin(getHBaseConfiguration(conf, tableDesc.getMeta())); + + HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc); + hAdmin.disableTable(hTableDesc.getName()); + hAdmin.deleteTable(hTableDesc.getName()); + } + + @Override + public Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); + boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + + Column indexColumn = null; + for (int i = 0; i < isRowKeyMappings.length; i++) { + if (isRowKeyMappings[i]) { + if (indexColumn != null) { + //Currently only supports single rowkey. + return null; + } + indexColumn = tableDesc.getSchema().getColumn(i); + } + } + return new Column[]{indexColumn}; + } + + @Override + public List getSplits(String fragmentId, TableDesc tableDesc, + List indexPredications) throws IOException { + Configuration hconf = getHBaseConfiguration(conf, tableDesc.getMeta()); + HTable htable = new HTable(hconf, tableDesc.getMeta().getOption(META_TABLE_KEY)); + + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { + HRegionLocation regLoc = htable.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false); + if (null == regLoc) { + throw new IOException("Expecting at least one region."); + } + List fragments = new ArrayList(1); + Fragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), + HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc.getHostname()); + fragments.add(fragment); + return fragments; + } + + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); + byte[] startRow = HConstants.EMPTY_START_ROW; + byte[] stopRow = HConstants.EMPTY_END_ROW; + IndexPredication indexPredication = null; + if (indexPredications != null && !indexPredications.isEmpty()) { + // Currently only supports rowkey + indexPredication = indexPredications.get(0); + + if (indexPredication.getStartValue() != null) { + startRow = serialize(columnMapping, indexPredication, indexPredication.getStartValue()); + } + if (indexPredication.getStopValue() != null) { + stopRow = serialize(columnMapping, indexPredication, indexPredication.getStopValue()); + } + } + + HBaseAdmin hAdmin = new HBaseAdmin(conf); + Map serverLoadMap = new HashMap(); + + List fragments = new ArrayList(keys.getFirst().length); + for (int i = 0; i < keys.getFirst().length; i++) { + HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); + + // determine if the given start an stop key fall into the region + if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) + && (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) { + byte[] fragmentStart = (startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0) ? + keys.getFirst()[i] : startRow; + + byte[] fragmentStop = (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && + keys.getSecond()[i].length > 0 ? keys.getSecond()[i] : stopRow; + + String regionName = location.getRegionInfo().getRegionNameAsString(); + + ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); + if (serverLoad == null) { + serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); + serverLoadMap.put(location.getServerName(), serverLoad); + } + + HBaseFragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), + fragmentStart, fragmentStop, location.getHostname()); + + // get region size + boolean foundLength = false; + for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { + if (regionName.equals(Bytes.toString(entry.getKey()))) { + RegionLoad regionLoad = entry.getValue(); + long storeFileSize = regionLoad.getStorefileSizeMB(); + fragment.setLength(storeFileSize); + foundLength = true; + break; + } + } + + if (!foundLength) { + fragment.setLength(TajoConstants.UNKNOWN_LENGTH); + } + + fragments.add(fragment); + if (LOG.isDebugEnabled()) { + LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + } + } + } + + if (!fragments.isEmpty()) { + ((HBaseFragment)fragments.get(fragments.size() - 1)).setLast(true); + } + return fragments; + } + + public static RowKeyMapping getRowKeyMapping(String cfName, String columnName) { + if (columnName == null || columnName.isEmpty()) { + return null; + } + + String[] tokens = columnName.split("#"); + if (!tokens[0].equalsIgnoreCase(ROWKEY_COLUMN_MAPPING)) { + return null; + } + + RowKeyMapping rowKeyMapping = new RowKeyMapping(); + + if (tokens.length == 2 && "b".equals(tokens[1])) { + rowKeyMapping.setBinary(true); + } + + if (cfName != null && !cfName.isEmpty()) { + rowKeyMapping.setKeyFieldIndex(Integer.parseInt(cfName)); + } + return rowKeyMapping; + } + + private byte[] serialize(ColumnMapping columnMapping, + IndexPredication indexPredication, Datum datum) throws IOException { + if (columnMapping.getIsBinaryColumns()[indexPredication.getColumnId()]) { + return HBaseBinarySerializerDeserializer.serialize(indexPredication.getColumn(), datum); + } else { + return HBaseTextSerializerDeserializer.serialize(indexPredication.getColumn(), datum); + } + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java new file mode 100644 index 0000000000..89ffb39eb8 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.storage.SerializerDeserializer; +import org.apache.tajo.util.NumberUtil; + +import java.io.IOException; +import java.io.OutputStream; + +public class HBaseTextSerializerDeserializer implements SerializerDeserializer { + + @Override + public int serialize(Column col, Datum datum, OutputStream out, byte[] nullCharacters) throws IOException { + return 0; + } + + @Override + public Datum deserialize(Column col, byte[] bytes, int offset, int length, byte[] nullCharacters) throws IOException { + Datum datum; + switch (col.getDataType().getType()) { + case INT1: + case INT2: + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt2((short) NumberUtil.parseInt(bytes, offset, length)); + break; + case INT4: + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt4(NumberUtil.parseInt(bytes, offset, length)); + break; + case INT8: + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(new String(bytes, offset, length)); + break; + case FLOAT4: + datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat4(new String(bytes, offset, length)); + break; + case FLOAT8: + datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat8(NumberUtil.parseDouble(bytes, offset, length)); + break; + case TEXT: + datum = bytes == null ? NullDatum.get() : DatumFactory.createText(bytes); + break; + default: + datum = NullDatum.get(); + break; + } + return datum; + } + + public static byte[] serialize(Column col, Datum datum) throws IOException { + if (datum == null || datum instanceof NullDatum) { + return null; + } + + return datum.asChars().getBytes(); + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java new file mode 100644 index 0000000000..4577703d53 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +public class RowKeyMapping { + private boolean isBinary; + private int keyFieldIndex; + + public boolean isBinary() { + return isBinary; + } + + public void setBinary(boolean isBinary) { + this.isBinary = isBinary; + } + + public int getKeyFieldIndex() { + return keyFieldIndex; + } + + public void setKeyFieldIndex(int keyFieldIndex) { + this.keyFieldIndex = keyFieldIndex; + } +} diff --git a/tajo-storage/src/main/proto/StorageFragmentProtos.proto b/tajo-storage/src/main/proto/StorageFragmentProtos.proto new file mode 100644 index 0000000000..dd79d74b6e --- /dev/null +++ b/tajo-storage/src/main/proto/StorageFragmentProtos.proto @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.tajo.storage.fragment"; +option java_outer_classname = "StorageFragmentProtos"; +option optimize_for = SPEED; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +import "CatalogProtos.proto"; + +message HBaseFragmentProto { + required string tableName = 1; + required string hbaseTableName = 2; + required bytes startRow = 3; + required bytes stopRow = 4; + required bool last = 5; + required int64 length = 6; + optional string regionLocation = 7; +} \ No newline at end of file diff --git a/tajo-storage/src/main/resources/storage-default.xml b/tajo-storage/src/main/resources/storage-default.xml index 4669477ea8..d3096d034f 100644 --- a/tajo-storage/src/main/resources/storage-default.xml +++ b/tajo-storage/src/main/resources/storage-default.xml @@ -20,27 +20,10 @@ --> - - tajo.storage.manager.v2 - false - - - - tajo.storage.manager.maxReadBytes - 8388608 - - - - - tajo.storage.manager.concurrency.perDisk - 1 - - - tajo.storage.scanner-handler - csv,raw,rcfile,row,trevni,parquet,sequencefile,avro + csv,raw,rcfile,row,trevni,parquet,sequencefile,avro,hbase @@ -76,6 +59,10 @@ tajo.storage.fragment.avro.class org.apache.tajo.storage.fragment.FileFragment + + tajo.storage.fragment.hbase.class + org.apache.tajo.storage.hbase.HBaseFragment + @@ -83,79 +70,44 @@ org.apache.tajo.storage.CSVFile$CSVScanner - - tajo.storage.scanner-handler.v2.csv.class - org.apache.tajo.storage.v2.CSVFileScanner - - tajo.storage.scanner-handler.raw.class org.apache.tajo.storage.RawFile$RawFileScanner - - tajo.storage.scanner-handler.v2.raw.class - org.apache.tajo.storage.RawFile$RawFileScanner - - tajo.storage.scanner-handler.rcfile.class org.apache.tajo.storage.rcfile.RCFile$RCFileScanner - - tajo.storage.scanner-handler.v2.rcfile.class - org.apache.tajo.storage.v2.RCFileScanner - - tajo.storage.scanner-handler.rowfile.class org.apache.tajo.storage.RowFile$RowFileScanner - - tajo.storage.scanner-handler.v2.rowfile.class - org.apache.tajo.storage.RowFile$RowFileScanner - - tajo.storage.scanner-handler.trevni.class org.apache.tajo.storage.trevni.TrevniScanner - - tajo.storage.scanner-handler.v2.trevni.class - org.apache.tajo.storage.trevni.TrevniScanner - - tajo.storage.scanner-handler.parquet.class org.apache.tajo.storage.parquet.ParquetScanner - - tajo.storage.scanner-handler.v2.parquet.class - org.apache.tajo.storage.parquet.ParquetScanner - - tajo.storage.scanner-handler.sequencefile.class org.apache.tajo.storage.sequencefile.SequenceFileScanner - - tajo.storage.scanner-handler.v2.sequencefile.class - org.apache.tajo.storage.sequencefile.SequenceFileScanner - - tajo.storage.scanner-handler.avro.class org.apache.tajo.storage.avro.AvroScanner - tajo.storage.scanner-handler.v2.avro.class - org.apache.tajo.storage.avro.AvroScanner + tajo.storage.scanner-handler.hbase.class + org.apache.tajo.storage.hbase.HBaseScanner @@ -203,4 +155,9 @@ tajo.storage.appender-handler.avro.class org.apache.tajo.storage.avro.AvroAppender + + + tajo.storage.appender-handler.hbase.class + org.apache.tajo.storage.hbase.HBaseAppender + From 6b9e57c761dd613b0455824452de27e2bcee930f Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Sat, 25 Oct 2014 01:42:39 +0900 Subject: [PATCH 03/29] TAJO-1127: Implements HBaseStorageManager Implements non-forward query. --- .../engine/planner/PhysicalPlannerImpl.java | 10 +- .../tajo/engine/planner/PlannerUtil.java | 92 ------ .../planner/PreLogicalPlanVerifier.java | 5 + .../org/apache/tajo/master/GlobalEngine.java | 6 + .../apache/tajo/master/LazyTaskScheduler.java | 13 +- .../master/NonForwardQueryResultScanner.java | 21 +- .../tajo/engine/planner/TestPlannerUtil.java | 12 +- .../tajo/engine/query/TestHBaseTable.java | 286 ++++++++++++------ .../queries/TestHBaseTable/testCATS.sql | 4 + .../results/TestHBaseTable/testJoin.result | 7 + .../TestHBaseTable/testNonForwardQuery.result | 102 +++++++ .../tajo/storage/FileStorageManager.java | 96 ++++++ .../org/apache/tajo/storage/MergeScanner.java | 5 +- .../apache/tajo/storage/StorageManager.java | 12 + .../storage/fragment/FragmentConvertor.java | 7 - .../tajo/storage/hbase/HBaseAppender.java | 62 ---- .../tajo/storage/hbase/HBaseFragment.java | 1 + .../storage/hbase/HBaseStorageManager.java | 77 ++++- 18 files changed, 529 insertions(+), 289 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 7f80a54f23..20bec17202 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -245,14 +245,6 @@ private PhysicalExec createPlanRecursive(TaskAttemptContext ctx, LogicalNode log } } - public static long getFragmentLength(TajoConf conf, Fragment fragment) { - if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { - return conf.getLongVar(ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); - } else { - return fragment.getLength(); - } - } - @VisibleForTesting public long estimateSizeRecursive(TaskAttemptContext ctx, String [] tableIds) throws IOException { long size = 0; @@ -260,7 +252,7 @@ public long estimateSizeRecursive(TaskAttemptContext ctx, String [] tableIds) th FragmentProto[] fragmentProtos = ctx.getTables(tableId); List fragments = FragmentConvertor.convert(ctx.getConf(), fragmentProtos); for (Fragment frag : fragments) { - size += getFragmentLength(ctx.getConf(), frag); + size += StorageManager.getFragmentLength(ctx.getConf(), frag); } } return size; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java index 6365e9e5b8..e74542519a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java @@ -815,98 +815,6 @@ public static String buildExplainString(LogicalNode node) { return explains.toString(); } - /** - * Listing table data file which is not empty. - * If the table is a partitioned table, return file list which has same partition key. - * @param tajoConf - * @param tableDesc - * @param fileIndex - * @param numResultFiles - * @return - * @throws IOException - */ - public static FragmentProto[] getNonZeroLengthDataFiles(TajoConf tajoConf,TableDesc tableDesc, - int fileIndex, int numResultFiles) throws IOException { - FileSystem fs = tableDesc.getPath().getFileSystem(tajoConf); - - List nonZeroLengthFiles = new ArrayList(); - if (fs.exists(tableDesc.getPath())) { - getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, fileIndex, numResultFiles, - new AtomicInteger(0)); - } - - List fragments = new ArrayList(); - - //In the case of partitioned table, return same partition key data files. - int numPartitionColumns = 0; - if (tableDesc.hasPartition()) { - numPartitionColumns = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size(); - } - String[] previousPartitionPathNames = null; - for (FileStatus eachFile: nonZeroLengthFiles) { - FileFragment fileFragment = new FileFragment(tableDesc.getName(), eachFile.getPath(), 0, eachFile.getLen(), null); - - if (numPartitionColumns > 0) { - // finding partition key; - Path filePath = fileFragment.getPath(); - Path parentPath = filePath; - String[] parentPathNames = new String[numPartitionColumns]; - for (int i = 0; i < numPartitionColumns; i++) { - parentPath = parentPath.getParent(); - parentPathNames[numPartitionColumns - i - 1] = parentPath.getName(); - } - - // If current partitionKey == previousPartitionKey, add to result. - if (previousPartitionPathNames == null) { - fragments.add(fileFragment); - } else if (previousPartitionPathNames != null && Arrays.equals(previousPartitionPathNames, parentPathNames)) { - fragments.add(fileFragment); - } else { - break; - } - previousPartitionPathNames = parentPathNames; - } else { - fragments.add(fileFragment); - } - } - return FragmentConvertor.toFragmentProtoArray(fragments.toArray(new FileFragment[]{})); - } - - private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List result, - int startFileIndex, int numResultFiles, - AtomicInteger currentFileIndex) throws IOException { - if (fs.isDirectory(path)) { - FileStatus[] files = fs.listStatus(path, FileStorageManager.hiddenFileFilter); - if (files != null && files.length > 0) { - for (FileStatus eachFile : files) { - if (result.size() >= numResultFiles) { - return; - } - if (eachFile.isDirectory()) { - getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles, - currentFileIndex); - } else if (eachFile.isFile() && eachFile.getLen() > 0) { - if (currentFileIndex.get() >= startFileIndex) { - result.add(eachFile); - } - currentFileIndex.incrementAndGet(); - } - } - } - } else { - FileStatus fileStatus = fs.getFileStatus(path); - if (fileStatus != null && fileStatus.getLen() > 0) { - if (currentFileIndex.get() >= startFileIndex) { - result.add(fileStatus); - } - currentFileIndex.incrementAndGet(); - if (result.size() >= numResultFiles) { - return; - } - } - } - } - public static List getIndexPredications(StorageManager storageHandler, TableDesc tableDesc, ScanNode scan) throws IOException { List indexPredications = new ArrayList(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java index f6d5540320..2e11aab6fe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java @@ -27,6 +27,7 @@ import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.util.TUtil; +import java.io.IOException; import java.util.Set; import java.util.Stack; @@ -272,6 +273,10 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws context.state.addVerification(String.format("relation \"%s\" does not exist", qualifiedName)); return null; } + if (!PlannerUtil.isFileStorageType(table.getMeta().getStoreType())) { + context.state.addVerification("Inserting into non-file storage is not supported."); + return null; + } if (table.hasPartition()) { int columnSize = table.getSchema().getColumns().size(); columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 2bcd301970..9ef3c196be 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -296,6 +296,12 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, responseBuilder.setResultCode(ClientProtos.ResultCode.OK); } } else { // it requires distributed execution. So, the query is forwarded to a query master. + if (rootNode.getChild().getType() == NodeType.CREATE_TABLE) { + StoreType storeType = ((CreateTableNode)rootNode.getChild()).getStorageType(); + if (!StorageManager.getStorageManager(context.getConf(), storeType).canCreateAsSelect(storeType)) { + throw new VerifyException("Inserting into non-file storage is not supported."); + } + } context.getSystemMetrics().counter("Query", "numDMLQuery").inc(); hookManager.doHooks(queryContext, plan); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java index 26647295fd..aff4b7d7b1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java @@ -40,6 +40,7 @@ import org.apache.tajo.master.querymaster.QueryUnit; import org.apache.tajo.master.querymaster.QueryUnitAttempt; import org.apache.tajo.master.querymaster.SubQuery; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; @@ -382,13 +383,13 @@ private void assignLeafTasks(List taskRequests) { } if (assignedFragmentSize + - PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()) > taskSize) { + StorageManager.getFragmentLength(conf, fragmentPair.getLeftFragment()) > taskSize) { break; } else { fragmentPairs.add(fragmentPair); - assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()); + assignedFragmentSize += StorageManager.getFragmentLength(conf, fragmentPair.getLeftFragment()); if (fragmentPair.getRightFragment() != null) { - assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getRightFragment()); + assignedFragmentSize += StorageManager.getFragmentLength(conf, fragmentPair.getRightFragment()); } } scheduledFragments.removeFragment(fragmentPair); @@ -405,13 +406,13 @@ private void assignLeafTasks(List taskRequests) { } if (assignedFragmentSize + - PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()) > taskSize) { + StorageManager.getFragmentLength(conf, fragmentPair.getLeftFragment()) > taskSize) { break; } else { fragmentPairs.add(fragmentPair); - assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getLeftFragment()); + assignedFragmentSize += StorageManager.getFragmentLength(conf, fragmentPair.getLeftFragment()); if (fragmentPair.getRightFragment() != null) { - assignedFragmentSize += PhysicalPlannerImpl.getFragmentLength(conf, fragmentPair.getRightFragment()); + assignedFragmentSize += StorageManager.getFragmentLength(conf, fragmentPair.getRightFragment()); } } scheduledFragments.removeFragment(fragmentPair); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java index 41ba4749aa..84f74ab285 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java @@ -35,6 +35,9 @@ import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; @@ -42,7 +45,7 @@ import java.util.List; public class NonForwardQueryResultScanner { - private static final int MAX_FILE_NUM_PER_SCAN = 100; + private static final int MAX_FRAGMENT_NUM_PER_SCAN = 100; private QueryId queryId; private String sessionId; @@ -55,7 +58,7 @@ public class NonForwardQueryResultScanner { private TajoConf tajoConf; private ScanNode scanNode; - private int currentFileIndex = 0; + private int currentFragmentIndex = 0; public NonForwardQueryResultScanner(TajoConf tajoConf, String sessionId, QueryId queryId, @@ -77,22 +80,24 @@ public void init() throws IOException { } private void initSeqScanExec() throws IOException { - FragmentProto[] fragments = PlannerUtil.getNonZeroLengthDataFiles(tajoConf, tableDesc, - currentFileIndex, MAX_FILE_NUM_PER_SCAN); - if (fragments != null && fragments.length > 0) { + List fragments = StorageManager.getStorageManager(tajoConf, tableDesc.getMeta().getStoreType()) + .getNonForwardSplit(tableDesc, currentFragmentIndex, MAX_FRAGMENT_NUM_PER_SCAN); + + if (fragments != null && !fragments.isEmpty()) { + FragmentProto[] fragmentProtos = FragmentConvertor.toFragmentProtoArray(fragments.toArray(new Fragment[]{})); this.taskContext = new TaskAttemptContext( new QueryContext(tajoConf), null, new QueryUnitAttemptId(new QueryUnitId(new ExecutionBlockId(queryId, 1), 0), 0), - fragments, null); + fragmentProtos, null); try { // scanNode must be clone cause SeqScanExec change target in the case of a partitioned table. - scanExec = new SeqScanExec(taskContext, (ScanNode)scanNode.clone(), fragments); + scanExec = new SeqScanExec(taskContext, (ScanNode)scanNode.clone(), fragmentProtos); } catch (CloneNotSupportedException e) { throw new IOException(e.getMessage(), e); } scanExec.init(); - currentFileIndex += fragments.length; + currentFragmentIndex += fragments.size(); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index b10b5829bc..e4b5c4b94b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -29,15 +29,18 @@ import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.function.builtin.SumInt; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; @@ -352,18 +355,19 @@ public void testGetNonZeroLengthDataFiles() throws Exception { for (int i = 0; i <= 5; i++) { int start = i * fileNum; - FragmentProto[] fragments = - PlannerUtil.getNonZeroLengthDataFiles(util.getConfiguration(), tableDesc, start, fileNum); + List fragments = + StorageManager.getFileStorageManager(util.getConfiguration()).getNonForwardSplit(tableDesc, start, fileNum); assertNotNull(fragments); - numResultFiles += fragments.length; + FragmentProto[] fragmentProtos = FragmentConvertor.toFragmentProtoArray(fragments.toArray(new Fragment[]{})); + numResultFiles += fragmentProtos.length; int expectedSize = fileNum; if (i == 5) { //last expectedSize = expectedFiles.size() - (fileNum * 5); } - comparePath(expectedFiles, fragments, start, expectedSize); + comparePath(expectedFiles, fragmentProtos, start, expectedSize); } assertEquals(expectedFiles.size(), numResultFiles); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index baa04545b8..eb9e34044e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -40,108 +40,107 @@ import java.text.DecimalFormat; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; @Category(IntegrationTest.class) public class TestHBaseTable extends QueryTestCaseBase { -// @Test -// public void testVerifyCreateHBaseTableRequiredMeta() throws Exception { -// try { -// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + -// "USING hbase").close(); -// -// fail("hbase table must have 'table' meta"); -// } catch (Exception e) { -// assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); -// } -// -// try { -// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + -// "USING hbase " + -// "WITH ('table'='hbase_table')").close(); -// -// fail("hbase table must have 'columns' meta"); -// } catch (Exception e) { -// assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); -// } -// -// try { -// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + -// "USING hbase " + -// "WITH ('table'='hbase_table', 'columns'='col1:,col2:')").close(); -// -// fail("hbase table must have 'hbase.zookeeper.quorum' meta"); -// } catch (Exception e) { -// assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); -// } -// } -// -// @Test -// public void testCreateHBaseTable() throws Exception { -// String hostName = InetAddress.getLocalHost().getHostName(); -// String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); -// assertNotNull(zkPort); -// -// executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + -// "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " + -// "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + -// "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); -// -// assertTableExists("hbase_mapped_table1"); -// -// HTableDescriptor hTableDesc = testingCluster.getHBaseUtil().getTableDescriptor("hbase_table"); -// assertNotNull(hTableDesc); -// assertEquals("hbase_table", hTableDesc.getNameAsString()); -// -// HColumnDescriptor[] hColumns = hTableDesc.getColumnFamilies(); -// // col1 is mapped to rowkey -// assertEquals(2, hColumns.length); -// assertEquals("col2", hColumns[0].getNameAsString()); -// assertEquals("col3", hColumns[1].getNameAsString()); -// -// executeString("DROP TABLE hbase_mapped_table1 PURGE"); -// } -// -// @Test -// public void testCreateNotExistsExternalHBaseTable() throws Exception { -// String hostName = InetAddress.getLocalHost().getHostName(); -// String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); -// assertNotNull(zkPort); -// -// try { -// executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + -// "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " + -// "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + -// "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); -// fail("External table should be a existed table."); -// } catch (Exception e) { -// assertTrue(e.getMessage().indexOf("External table should be a existed table.") >= 0); -// } -// } -// -// @Test -// public void testCreateExternalHBaseTable() throws Exception { -// HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); -// hTableDesc.addFamily(new HColumnDescriptor("col1")); -// hTableDesc.addFamily(new HColumnDescriptor("col2")); -// hTableDesc.addFamily(new HColumnDescriptor("col3")); -// testingCluster.getHBaseUtil().createTable(hTableDesc); -// -// String hostName = InetAddress.getLocalHost().getHostName(); -// String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); -// assertNotNull(zkPort); -// -// executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + -// "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + -// "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + -// "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); -// -// assertTableExists("external_hbase_mapped_table"); -// -// executeString("DROP TABLE external_hbase_mapped_table PURGE"); -// } + @Test + public void testVerifyCreateHBaseTableRequiredMeta() throws Exception { + try { + executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + + "USING hbase").close(); + + fail("hbase table must have 'table' meta"); + } catch (Exception e) { + assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); + } + + try { + executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + + "USING hbase " + + "WITH ('table'='hbase_table')").close(); + + fail("hbase table must have 'columns' meta"); + } catch (Exception e) { + assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); + } + + try { + executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " + + "USING hbase " + + "WITH ('table'='hbase_table', 'columns'='col1:,col2:')").close(); + + fail("hbase table must have 'hbase.zookeeper.quorum' meta"); + } catch (Exception e) { + assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); + } + } + + @Test + public void testCreateHBaseTable() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table1"); + + HTableDescriptor hTableDesc = testingCluster.getHBaseUtil().getTableDescriptor("hbase_table"); + assertNotNull(hTableDesc); + assertEquals("hbase_table", hTableDesc.getNameAsString()); + + HColumnDescriptor[] hColumns = hTableDesc.getColumnFamilies(); + // col1 is mapped to rowkey + assertEquals(2, hColumns.length); + assertEquals("col2", hColumns[0].getNameAsString()); + assertEquals("col3", hColumns[1].getNameAsString()); + + executeString("DROP TABLE hbase_mapped_table1 PURGE"); + } + + @Test + public void testCreateNotExistsExternalHBaseTable() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + try { + executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + fail("External table should be a existed table."); + } catch (Exception e) { + assertTrue(e.getMessage().indexOf("External table should be a existed table.") >= 0); + } + } + + @Test + public void testCreateExternalHBaseTable() throws Exception { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col1")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("external_hbase_mapped_table"); + + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } @Test public void testSimpleSelectQuery() throws Exception { @@ -321,5 +320,96 @@ public void testIndexPredication() throws Exception { ResultSet res = executeString("select * from external_hbase_mapped_table where rk >= '020' and rk <= '055'"); assertResultSet(res); res.close(); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } + + @Test + public void testNonForwardQuery() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + "'hbase.split.rowkeys'='010,040,060,080', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + + assertTableExists("external_hbase_mapped_table"); + HBaseAdmin hAdmin = new HBaseAdmin(HBaseConfiguration.create(conf)); + hAdmin.tableExists("external_hbase_table"); + + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + assertEquals(5, keys.getFirst().length); + + DecimalFormat df = new DecimalFormat("000"); + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(df.format(i)).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table"); + assertResultSet(res); + res.close(); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } + + @Test + public void testJoin() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 int8) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " + + "'hbase.split.rowkeys'='010,040,060,080', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + + assertTableExists("external_hbase_mapped_table"); + HBaseAdmin hAdmin = new HBaseAdmin(HBaseConfiguration.create(conf)); + hAdmin.tableExists("external_hbase_table"); + + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + assertEquals(5, keys.getFirst().length); + + DecimalFormat df = new DecimalFormat("000"); + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(df.format(i)).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes((long)i)); + htable.put(put); + } + + ResultSet res = executeString("select a.rk, a.col1, a.col2, a.col3, b.l_orderkey, b.l_linestatus " + + "from external_hbase_mapped_table a " + + "join default.lineitem b on a.col3 = b.l_orderkey"); + assertResultSet(res); + res.close(); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } + + @Test + public void testCATS() throws Exception { + try { + ResultSet res = executeQuery(); + fail("CATS not supported"); + } catch (Exception e) { + if (e.getMessage().indexOf("not supported") < 0) { + fail(e.getMessage()); + } + } } } diff --git a/tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql b/tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql new file mode 100644 index 0000000000..051d360823 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql @@ -0,0 +1,4 @@ +create table hbase_mapped_table (rk int8, col1 int8) +using hbase with ('table'='hbase_table', 'columns'=':key,col1:a#b') as +select l_orderkey, l_partkey +from default.lineitem diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result new file mode 100644 index 0000000000..fe4fcaebc3 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result @@ -0,0 +1,7 @@ +rk,col1,col2,col3,l_orderkey,l_linestatus +------------------------------- +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O +002,a-2,{"k1":"k1-2", "k2":"k2-2"},2,2,O +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result new file mode 100644 index 0000000000..799dff5aea --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result @@ -0,0 +1,102 @@ +rk,col1,col2,col3 +------------------------------- +000,a-0,{"k1":"k1-0", "k2":"k2-0"},b-0 +001,a-1,{"k1":"k1-1", "k2":"k2-1"},b-1 +002,a-2,{"k1":"k1-2", "k2":"k2-2"},b-2 +003,a-3,{"k1":"k1-3", "k2":"k2-3"},b-3 +004,a-4,{"k1":"k1-4", "k2":"k2-4"},b-4 +005,a-5,{"k1":"k1-5", "k2":"k2-5"},b-5 +006,a-6,{"k1":"k1-6", "k2":"k2-6"},b-6 +007,a-7,{"k1":"k1-7", "k2":"k2-7"},b-7 +008,a-8,{"k1":"k1-8", "k2":"k2-8"},b-8 +009,a-9,{"k1":"k1-9", "k2":"k2-9"},b-9 +010,a-10,{"k1":"k1-10", "k2":"k2-10"},b-10 +011,a-11,{"k1":"k1-11", "k2":"k2-11"},b-11 +012,a-12,{"k1":"k1-12", "k2":"k2-12"},b-12 +013,a-13,{"k1":"k1-13", "k2":"k2-13"},b-13 +014,a-14,{"k1":"k1-14", "k2":"k2-14"},b-14 +015,a-15,{"k1":"k1-15", "k2":"k2-15"},b-15 +016,a-16,{"k1":"k1-16", "k2":"k2-16"},b-16 +017,a-17,{"k1":"k1-17", "k2":"k2-17"},b-17 +018,a-18,{"k1":"k1-18", "k2":"k2-18"},b-18 +019,a-19,{"k1":"k1-19", "k2":"k2-19"},b-19 +020,a-20,{"k1":"k1-20", "k2":"k2-20"},b-20 +021,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21 +022,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22 +023,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23 +024,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24 +025,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25 +026,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26 +027,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27 +028,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28 +029,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29 +030,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30 +031,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31 +032,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32 +033,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33 +034,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34 +035,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35 +036,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36 +037,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37 +038,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38 +039,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39 +040,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40 +041,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41 +042,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42 +043,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43 +044,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44 +045,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45 +046,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46 +047,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47 +048,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48 +049,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49 +050,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50 +051,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51 +052,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52 +053,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53 +054,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54 +055,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55 +056,a-56,{"k1":"k1-56", "k2":"k2-56"},b-56 +057,a-57,{"k1":"k1-57", "k2":"k2-57"},b-57 +058,a-58,{"k1":"k1-58", "k2":"k2-58"},b-58 +059,a-59,{"k1":"k1-59", "k2":"k2-59"},b-59 +060,a-60,{"k1":"k1-60", "k2":"k2-60"},b-60 +061,a-61,{"k1":"k1-61", "k2":"k2-61"},b-61 +062,a-62,{"k1":"k1-62", "k2":"k2-62"},b-62 +063,a-63,{"k1":"k1-63", "k2":"k2-63"},b-63 +064,a-64,{"k1":"k1-64", "k2":"k2-64"},b-64 +065,a-65,{"k1":"k1-65", "k2":"k2-65"},b-65 +066,a-66,{"k1":"k1-66", "k2":"k2-66"},b-66 +067,a-67,{"k1":"k1-67", "k2":"k2-67"},b-67 +068,a-68,{"k1":"k1-68", "k2":"k2-68"},b-68 +069,a-69,{"k1":"k1-69", "k2":"k2-69"},b-69 +070,a-70,{"k1":"k1-70", "k2":"k2-70"},b-70 +071,a-71,{"k1":"k1-71", "k2":"k2-71"},b-71 +072,a-72,{"k1":"k1-72", "k2":"k2-72"},b-72 +073,a-73,{"k1":"k1-73", "k2":"k2-73"},b-73 +074,a-74,{"k1":"k1-74", "k2":"k2-74"},b-74 +075,a-75,{"k1":"k1-75", "k2":"k2-75"},b-75 +076,a-76,{"k1":"k1-76", "k2":"k2-76"},b-76 +077,a-77,{"k1":"k1-77", "k2":"k2-77"},b-77 +078,a-78,{"k1":"k1-78", "k2":"k2-78"},b-78 +079,a-79,{"k1":"k1-79", "k2":"k2-79"},b-79 +080,a-80,{"k1":"k1-80", "k2":"k2-80"},b-80 +081,a-81,{"k1":"k1-81", "k2":"k2-81"},b-81 +082,a-82,{"k1":"k1-82", "k2":"k2-82"},b-82 +083,a-83,{"k1":"k1-83", "k2":"k2-83"},b-83 +084,a-84,{"k1":"k1-84", "k2":"k2-84"},b-84 +085,a-85,{"k1":"k1-85", "k2":"k2-85"},b-85 +086,a-86,{"k1":"k1-86", "k2":"k2-86"},b-86 +087,a-87,{"k1":"k1-87", "k2":"k2-87"},b-87 +088,a-88,{"k1":"k1-88", "k2":"k2-88"},b-88 +089,a-89,{"k1":"k1-89", "k2":"k2-89"},b-89 +090,a-90,{"k1":"k1-90", "k2":"k2-90"},b-90 +091,a-91,{"k1":"k1-91", "k2":"k2-91"},b-91 +092,a-92,{"k1":"k1-92", "k2":"k2-92"},b-92 +093,a-93,{"k1":"k1-93", "k2":"k2-93"},b-93 +094,a-94,{"k1":"k1-94", "k2":"k2-94"},b-94 +095,a-95,{"k1":"k1-95", "k2":"k2-95"},b-95 +096,a-96,{"k1":"k1-96", "k2":"k2-96"},b-96 +097,a-97,{"k1":"k1-97", "k2":"k2-97"},b-97 +098,a-98,{"k1":"k1-98", "k2":"k2-98"},b-98 +099,a-99,{"k1":"k1-99", "k2":"k2-99"},b-99 \ No newline at end of file diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index d3b18cf672..33ae023b69 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -30,6 +30,7 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; @@ -43,6 +44,7 @@ import java.lang.reflect.Constructor; import java.net.URI; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; public class FileStorageManager extends StorageManager { private final Log LOG = LogFactory.getLog(FileStorageManager.class); @@ -701,4 +703,98 @@ public void purgeTable(TableDesc tableDesc) throws IOException { public Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { return null; } + + @Override + public List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { + // Listing table data file which is not empty. + // If the table is a partitioned table, return file list which has same partition key. + FileSystem fs = tableDesc.getPath().getFileSystem(conf); + + List nonZeroLengthFiles = new ArrayList(); + if (fs.exists(tableDesc.getPath())) { + getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, currentPage, numFragments, + new AtomicInteger(0)); + } + + List fragments = new ArrayList(); + + //In the case of partitioned table, return same partition key data files. + int numPartitionColumns = 0; + if (tableDesc.hasPartition()) { + numPartitionColumns = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size(); + } + String[] previousPartitionPathNames = null; + for (FileStatus eachFile: nonZeroLengthFiles) { + FileFragment fileFragment = new FileFragment(tableDesc.getName(), eachFile.getPath(), 0, eachFile.getLen(), null); + + if (numPartitionColumns > 0) { + // finding partition key; + Path filePath = fileFragment.getPath(); + Path parentPath = filePath; + String[] parentPathNames = new String[numPartitionColumns]; + for (int i = 0; i < numPartitionColumns; i++) { + parentPath = parentPath.getParent(); + parentPathNames[numPartitionColumns - i - 1] = parentPath.getName(); + } + + // If current partitionKey == previousPartitionKey, add to result. + if (previousPartitionPathNames == null) { + fragments.add(fileFragment); + } else if (previousPartitionPathNames != null && Arrays.equals(previousPartitionPathNames, parentPathNames)) { + fragments.add(fileFragment); + } else { + break; + } + previousPartitionPathNames = parentPathNames; + } else { + fragments.add(fileFragment); + } + } + + return fragments; + } + + private void getNonZeroLengthDataFiles(FileSystem fs, Path path, List result, + int startFileIndex, int numResultFiles, + AtomicInteger currentFileIndex) throws IOException { + if (fs.isDirectory(path)) { + FileStatus[] files = fs.listStatus(path, FileStorageManager.hiddenFileFilter); + if (files != null && files.length > 0) { + for (FileStatus eachFile : files) { + if (result.size() >= numResultFiles) { + return; + } + if (eachFile.isDirectory()) { + getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles, + currentFileIndex); + } else if (eachFile.isFile() && eachFile.getLen() > 0) { + if (currentFileIndex.get() >= startFileIndex) { + result.add(eachFile); + } + currentFileIndex.incrementAndGet(); + } + } + } + } else { + FileStatus fileStatus = fs.getFileStatus(path); + if (fileStatus != null && fileStatus.getLen() > 0) { + if (currentFileIndex.get() >= startFileIndex) { + result.add(fileStatus); + } + currentFileIndex.incrementAndGet(); + if (result.size() >= numResultFiles) { + return; + } + } + } + } + + @Override + public boolean canCreateAsSelect(StoreType storeType) { + if (storeType == StoreType.RAW) { + return false; + } else { + return true; + } + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java index 89e59d08b7..78b9d3dd0b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -64,8 +64,9 @@ public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List 0) { - numBytes += eachFileFragment.getLength(); + long fragmentLength = StorageManager.getFragmentLength((TajoConf)conf, eachFileFragment); + if (fragmentLength > 0) { + numBytes += fragmentLength; fragments.add(eachFileFragment); } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index b8a8034839..14aeef2ac8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; @@ -83,7 +84,10 @@ public abstract class StorageManager { public abstract void purgeTable(TableDesc tableDesc) throws IOException; public abstract List getSplits(String fragmentId, TableDesc tableDesc, List indexPredications) throws IOException; + public abstract List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) + throws IOException; public abstract Column[] getIndexableColumns(TableDesc tableDesc) throws IOException; + public abstract boolean canCreateAsSelect(StoreType storeType); public void init(TajoConf tajoConf) throws IOException { this.conf = tajoConf; @@ -273,4 +277,12 @@ public static synchronized SeekableScanner getSeekableScanner( return getSeekableScanner(conf, meta, schema, fragment, schema); } + + public static long getFragmentLength(TajoConf conf, Fragment fragment) { + if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { + return conf.getLongVar(ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); + } else { + return fragment.getLength(); + } + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java index 0f5802da7a..07720c708c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java @@ -30,7 +30,6 @@ import java.util.Map; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; -import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @ThreadSafe public class FragmentConvertor { @@ -47,12 +46,6 @@ public class FragmentConvertor { */ private static final Class[] DEFAULT_FRAGMENT_PARAMS = { ByteString.class }; - public static Class getFragmentClass(Configuration conf, StoreType storeType) - throws IOException { - String name = storeType.name().toLowerCase(); - return getFragmentClass(conf, name); - } - public static Class getFragmentClass(Configuration conf, String storeType) throws IOException { Class fragmentClass = CACHED_FRAGMENT_CLASSES.get(storeType.toLowerCase()); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java deleted file mode 100644 index 54f69dfe78..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.hbase; - -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.Tuple; - -import java.io.IOException; - -public class HBaseAppender implements Appender { - @Override - public void init() throws IOException { - - } - - @Override - public void addTuple(Tuple t) throws IOException { - - } - - @Override - public void flush() throws IOException { - - } - - @Override - public long getEstimatedOutputSize() throws IOException { - return 0; - } - - @Override - public void close() throws IOException { - - } - - @Override - public void enableStats() { - - } - - @Override - public TableStats getStats() { - return null; - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java index 225916cc23..52c5e7abc1 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -66,6 +66,7 @@ private void init(HBaseFragmentProto proto) { this.startRow = proto.getStartRow().toByteArray(); this.stopRow = proto.getStopRow().toByteArray(); this.regionLocation = proto.getRegionLocation(); + this.length = proto.getLength(); this.last = proto.getLast(); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 01f394cc04..7de0cabd7d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -30,6 +30,7 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -342,7 +343,7 @@ public List getSplits(String fragmentId, TableDesc tableDesc, for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { if (regionName.equals(Bytes.toString(entry.getKey()))) { RegionLoad regionLoad = entry.getValue(); - long storeFileSize = regionLoad.getStorefileSizeMB(); + long storeFileSize = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; fragment.setLength(storeFileSize); foundLength = true; break; @@ -396,4 +397,78 @@ private byte[] serialize(ColumnMapping columnMapping, return HBaseTextSerializerDeserializer.serialize(indexPredication.getColumn(), datum); } } + + @Override + public List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) + throws IOException { + Configuration hconf = getHBaseConfiguration(conf, tableDesc.getMeta()); + HTable htable = new HTable(hconf, tableDesc.getMeta().getOption(META_TABLE_KEY)); + + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { + return new ArrayList(1); + } + + HBaseAdmin hAdmin = new HBaseAdmin(conf); + Map serverLoadMap = new HashMap(); + + List fragments = new ArrayList(keys.getFirst().length); + + int start = currentPage * numFragments; + if (start >= keys.getFirst().length) { + return new ArrayList(1); + } + int end = (currentPage + 1) * numFragments; + if (end > keys.getFirst().length) { + end = keys.getFirst().length; + } + for (int i = start; i < end; i++) { + HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); + + String regionName = location.getRegionInfo().getRegionNameAsString(); + ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); + if (serverLoad == null) { + serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); + serverLoadMap.put(location.getServerName(), serverLoad); + } + + HBaseFragment fragment = new HBaseFragment(tableDesc.getName(), htable.getName().getNameAsString(), + location.getRegionInfo().getStartKey(), location.getRegionInfo().getEndKey(), location.getHostname()); + + // get region size + boolean foundLength = false; + for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { + if (regionName.equals(Bytes.toString(entry.getKey()))) { + RegionLoad regionLoad = entry.getValue(); + long storeLength = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; + if (storeLength == 0) { + // If store size is smaller than 1 MB, storeLength is zero + storeLength = 1 * 1024 * 1024; //default 1MB + } + fragment.setLength(storeLength); + foundLength = true; + break; + } + } + + if (!foundLength) { + fragment.setLength(TajoConstants.UNKNOWN_LENGTH); + } + + fragments.add(fragment); + if (LOG.isDebugEnabled()) { + LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + } + } + + if (!fragments.isEmpty()) { + ((HBaseFragment)fragments.get(fragments.size() - 1)).setLast(true); + } + return fragments; + } + + @Override + public boolean canCreateAsSelect(StoreType storeType) { + return false; + } } From eb2e8ecef5ac344d3286aa8d227b25489c6b4273 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Sat, 25 Oct 2014 19:12:59 +0900 Subject: [PATCH 04/29] TAJO-1127: Implements HBaseStorageManager. --- .../org/apache/tajo/HBaseTestClusterUtil.java | 51 +- .../org/apache/tajo/TajoTestingCluster.java | 14 +- .../tajo/engine/query/TestHBaseTable.java | 329 +++++++----- .../tajo/storage/FileStorageManager.java | 4 + .../org/apache/tajo/storage/MergeScanner.java | 3 + .../apache/tajo/storage/StorageManager.java | 9 + .../tajo/storage/hbase/HBaseScanner.java | 18 +- .../storage/hbase/HBaseStorageManager.java | 488 ++++++++++++------ 8 files changed, 592 insertions(+), 324 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java b/tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java index b322a2ef30..a8e4a5c38e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java @@ -18,6 +18,8 @@ package org.apache.tajo; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -37,6 +39,7 @@ import static org.apache.hadoop.hbase.HConstants.REPLICATION_ENABLE_KEY; public class HBaseTestClusterUtil { + private static final Log LOG = LogFactory.getLog(HBaseTestClusterUtil.class); private Configuration conf; private MiniHBaseCluster hbaseCluster; private MiniZooKeeperCluster zkCluster; @@ -73,19 +76,22 @@ public Path createRootDir() throws IOException { return hbaseRootdir; } - public void stopHBaseCluster() throws Exception { + public void stopHBaseCluster() throws IOException { if (hbaseCluster != null) { + LOG.info("MiniHBaseCluster stopped"); hbaseCluster.shutdown(); - } - - if (zkCluster != null) { - zkCluster.shutdown(); + hbaseCluster.waitUntilShutDown(); + hbaseCluster = null; } } public void startHBaseCluster() throws Exception { - File zkDataPath = new File(testBaseDir, "zk"); - startMiniZKCluster(zkDataPath); + if (zkCluster == null) { + startMiniZKCluster(); + } + if (hbaseCluster != null) { + return; + } System.setProperty("HBASE_ZNODE_FILE", testBaseDir + "/hbase_znode_file"); if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { @@ -109,16 +115,19 @@ public void startHBaseCluster() throws Exception { } s.close(); t.close(); + LOG.info("MiniHBaseCluster started"); + } /** * Start a mini ZK cluster. If the property "test.hbase.zookeeper.property.clientPort" is set * the port mentionned is used as the default port for ZooKeeper. */ - private MiniZooKeeperCluster startMiniZKCluster(final File dir) + public MiniZooKeeperCluster startMiniZKCluster() throws Exception { + File zkDataPath = new File(testBaseDir, "zk"); if (this.zkCluster != null) { - throw new IOException("Cluster already running at " + dir); + throw new IOException("Cluster already running at " + zkDataPath); } this.zkCluster = new MiniZooKeeperCluster(conf); final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0); @@ -126,11 +135,21 @@ private MiniZooKeeperCluster startMiniZKCluster(final File dir) // If there is a port in the config file, we use it. this.zkCluster.setDefaultClientPort(defPort); } - int clientPort = this.zkCluster.startup(dir, 1); + int clientPort = this.zkCluster.startup(zkDataPath, 1); this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); + LOG.info("MiniZooKeeperCluster started"); + return this.zkCluster; } + public void stopZooKeeperCluster() throws IOException { + if (zkCluster != null) { + LOG.info("MiniZooKeeperCluster stopped"); + zkCluster.shutdown(); + zkCluster = null; + } + } + public Configuration getConf() { return conf; } @@ -145,11 +164,19 @@ public MiniHBaseCluster getMiniHBaseCluster() { public HTableDescriptor getTableDescriptor(String tableName) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); - return admin.getTableDescriptor(Bytes.toBytes(tableName)); + try { + return admin.getTableDescriptor(Bytes.toBytes(tableName)); + } finally { + admin.close(); + } } public void createTable(HTableDescriptor hTableDesc) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); - admin.createTable(hTableDesc); + try { + admin.createTable(hTableDesc); + } finally { + admin.close(); + } } } diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 4544959b00..a20b8ab2bb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -470,7 +470,6 @@ public void startMiniCluster(final int numSlaves, final String [] dataNodeHosts) this.dfsCluster.waitClusterUp(); hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir); - hbaseUtil.startHBaseCluster(); if(!standbyWorkerMode) { startMiniYarnCluster(); @@ -559,7 +558,6 @@ public void shutdownMiniCluster() throws IOException { } if(this.dfsCluster != null) { - try { FileSystem fs = this.dfsCluster.getFileSystem(); if (fs != null) fs.close(); @@ -569,14 +567,6 @@ public void shutdownMiniCluster() throws IOException { } } - if (this.hbaseUtil != null) { - try { - this.hbaseUtil.stopHBaseCluster(); - } catch (Exception e) { - System.err.println("error stopping hbase cluster: " + e); - } - } - if(this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) { if(!ShutdownHookManager.get().isShutdownInProgress()) { //TODO clean test dir when ShutdownInProgress @@ -586,6 +576,10 @@ public void shutdownMiniCluster() throws IOException { } this.clusterTestBuildDir = null; } + + hbaseUtil.stopZooKeeperCluster(); + hbaseUtil.stopHBaseCluster(); + LOG.info("Minicluster is down"); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index eb9e34044e..b15ca13634 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -18,6 +18,8 @@ package org.apache.tajo.engine.query; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.tajo.IntegrationTest; @@ -31,7 +33,10 @@ import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.hbase.HBaseFragment; +import org.apache.tajo.storage.hbase.HBaseStorageManager; import org.apache.tajo.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -44,6 +49,25 @@ @Category(IntegrationTest.class) public class TestHBaseTable extends QueryTestCaseBase { + private static final Log LOG = LogFactory.getLog(TestHBaseTable.class); + + @BeforeClass + public static void beforeClass() { + try { + testingCluster.getHBaseUtil().startHBaseCluster(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + testingCluster.getHBaseUtil().stopHBaseCluster(); + } catch (Exception e) { + e.printStackTrace(); + } + } @Test public void testVerifyCreateHBaseTableRequiredMeta() throws Exception { @@ -161,22 +185,28 @@ public void testSimpleSelectQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); + HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE)) + .getConnection(testingCluster.getHBaseUtil().getConf()); + HTableInterface htable = hconn.getTable("external_hbase_table"); - HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); - for (int i = 0; i < 100; i++) { - Put put = new Put(String.valueOf(i).getBytes()); - put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); - put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - htable.put(put); - } + try { + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(i).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } - ResultSet res = executeString("select * from external_hbase_mapped_table where rk > '20'"); - assertResultSet(res); - cleanupQuery(res); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + ResultSet res = executeString("select * from external_hbase_mapped_table where rk > '20'"); + assertResultSet(res); + cleanupQuery(res); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } finally { + htable.close(); + } } @Test @@ -198,35 +228,42 @@ public void testBinaryMappedQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); - for (int i = 0; i < 100; i++) { - Put put = new Put(Bytes.toBytes((long) i)); - put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); - put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes(i)); - htable.put(put); - } + HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE)) + .getConnection(testingCluster.getHBaseUtil().getConf()); + HTableInterface htable = hconn.getTable("external_hbase_table"); - ResultSet res = executeString("select * from external_hbase_mapped_table where rk > 20"); - assertResultSet(res); - res.close(); + try { + for (int i = 0; i < 100; i++) { + Put put = new Put(Bytes.toBytes((long) i)); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes(i)); + htable.put(put); + } - //Projection - res = executeString("select col3 from external_hbase_mapped_table where rk > 95"); + ResultSet res = executeString("select * from external_hbase_mapped_table where rk > 20"); + assertResultSet(res); + res.close(); - String expected = "col3\n" + - "-------------------------------\n" + - "96\n" + - "97\n" + - "98\n" + - "99\n"; + //Projection + res = executeString("select col3 from external_hbase_mapped_table where rk > 95"); - assertEquals(expected, resultSetToString(res)); - res.close(); + String expected = "col3\n" + + "-------------------------------\n" + + "96\n" + + "97\n" + + "98\n" + + "99\n"; - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + assertEquals(expected, resultSetToString(res)); + res.close(); + + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } finally { + htable.close(); + } } @Test @@ -247,17 +284,24 @@ public void testRowFieldSelectQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); - for (int i = 0; i < 100; i++) { - Put put = new Put(("field1-" + i + "_field2-" + i).getBytes()); - put.add("col3".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - htable.put(put); - } + HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE)) + .getConnection(testingCluster.getHBaseUtil().getConf()); + HTableInterface htable = hconn.getTable("external_hbase_table"); - ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 > 'field1-20'"); - assertResultSet(res); - cleanupQuery(res); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + try { + for (int i = 0; i < 100; i++) { + Put put = new Put(("field1-" + i + "_field2-" + i).getBytes()); + put.add("col3".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 > 'field1-20'"); + assertResultSet(res); + cleanupQuery(res); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } finally { + htable.close(); + } } @Test @@ -274,53 +318,58 @@ public void testIndexPredication() throws Exception { assertTableExists("external_hbase_mapped_table"); - HBaseAdmin hAdmin = new HBaseAdmin(HBaseConfiguration.create(conf)); + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); hAdmin.tableExists("external_hbase_table"); HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); - org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); - assertEquals(5, keys.getFirst().length); - - DecimalFormat df = new DecimalFormat("000"); - for (int i = 0; i < 100; i++) { - Put put = new Put(String.valueOf(df.format(i)).getBytes()); - put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); - put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - htable.put(put); - } - - TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "external_hbase_mapped_table"); - - // where rk >= '020' and rk <= '055' - ScanNode scanNode = new ScanNode(1); - EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("020"))); - EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("055"))); - EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); - - scanNode.setQual(evalNodeA); - - StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE); - List fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, - PlannerUtil.getIndexPredications(storageManager, tableDesc, scanNode)); - - assertEquals(2, fragments.size()); - HBaseFragment fragment1 = (HBaseFragment)fragments.get(0); - assertEquals("020", new String(fragment1.getStartRow())); - assertEquals("040", new String(fragment1.getStopRow())); - - HBaseFragment fragment2 = (HBaseFragment)fragments.get(1); - assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("055", new String(fragment2.getStopRow())); + try { + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + assertEquals(5, keys.getFirst().length); + + DecimalFormat df = new DecimalFormat("000"); + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(df.format(i)).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } - ResultSet res = executeString("select * from external_hbase_mapped_table where rk >= '020' and rk <= '055'"); - assertResultSet(res); - res.close(); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "external_hbase_mapped_table"); + + // where rk >= '020' and rk <= '055' + ScanNode scanNode = new ScanNode(1); + EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("020"))); + EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("055"))); + EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); + + scanNode.setQual(evalNodeA); + + StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE); + List fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, + PlannerUtil.getIndexPredications(storageManager, tableDesc, scanNode)); + + assertEquals(2, fragments.size()); + HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + HBaseFragment fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("055", new String(fragment2.getStopRow())); + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk >= '020' and rk <= '055'"); + assertResultSet(res); + res.close(); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } finally { + htable.close(); + hAdmin.close(); + } } @Test @@ -337,28 +386,35 @@ public void testNonForwardQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HBaseAdmin hAdmin = new HBaseAdmin(HBaseConfiguration.create(conf)); - hAdmin.tableExists("external_hbase_table"); + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); + HTable htable = null; + try { + hAdmin.tableExists("external_hbase_table"); + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + assertEquals(5, keys.getFirst().length); + + DecimalFormat df = new DecimalFormat("000"); + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(df.format(i)).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } - HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); - org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); - assertEquals(5, keys.getFirst().length); - - DecimalFormat df = new DecimalFormat("000"); - for (int i = 0; i < 100; i++) { - Put put = new Put(String.valueOf(df.format(i)).getBytes()); - put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); - put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - htable.put(put); + ResultSet res = executeString("select * from external_hbase_mapped_table"); + assertResultSet(res); + res.close(); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } finally { + hAdmin.close(); + if (htable == null) { + htable.close(); + } } - - ResultSet res = executeString("select * from external_hbase_mapped_table"); - assertResultSet(res); - res.close(); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); } @Test @@ -375,30 +431,37 @@ public void testJoin() throws Exception { assertTableExists("external_hbase_mapped_table"); - HBaseAdmin hAdmin = new HBaseAdmin(HBaseConfiguration.create(conf)); - hAdmin.tableExists("external_hbase_table"); + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); + HTable htable = null; + try { + hAdmin.tableExists("external_hbase_table"); + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + assertEquals(5, keys.getFirst().length); + + DecimalFormat df = new DecimalFormat("000"); + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(df.format(i)).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes((long) i)); + htable.put(put); + } - HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); - org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); - assertEquals(5, keys.getFirst().length); - - DecimalFormat df = new DecimalFormat("000"); - for (int i = 0; i < 100; i++) { - Put put = new Put(String.valueOf(df.format(i)).getBytes()); - put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); - put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes((long)i)); - htable.put(put); + ResultSet res = executeString("select a.rk, a.col1, a.col2, a.col3, b.l_orderkey, b.l_linestatus " + + "from external_hbase_mapped_table a " + + "join default.lineitem b on a.col3 = b.l_orderkey"); + assertResultSet(res); + res.close(); + executeString("DROP TABLE external_hbase_mapped_table PURGE"); + } finally { + hAdmin.close(); + if (htable != null) { + htable.close(); + } } - - ResultSet res = executeString("select a.rk, a.col1, a.col2, a.col3, b.l_orderkey, b.l_linestatus " + - "from external_hbase_mapped_table a " + - "join default.lineitem b on a.col3 = b.l_orderkey"); - assertResultSet(res); - res.close(); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); } @Test diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 33ae023b69..8815a74bbc 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -797,4 +797,8 @@ public boolean canCreateAsSelect(StoreType storeType) { return true; } } + + @Override + public void closeStorageManager() { + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java index 78b9d3dd0b..66b3667f3e 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -122,6 +122,9 @@ public Tuple next() throws IOException { @Override public void reset() throws IOException { this.iterator = fragments.iterator(); + if (currentScanner != null) { + currentScanner.close(); + } this.currentScanner = getNextScanner(); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 14aeef2ac8..a0b5957b3b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -88,12 +88,21 @@ public abstract List getNonForwardSplit(TableDesc tableDesc, int curre throws IOException; public abstract Column[] getIndexableColumns(TableDesc tableDesc) throws IOException; public abstract boolean canCreateAsSelect(StoreType storeType); + public abstract void closeStorageManager(); public void init(TajoConf tajoConf) throws IOException { this.conf = tajoConf; storageInit(); } + public void close() throws IOException { + synchronized(storageManagers) { + for (StorageManager eachStorageManager: storageManagers.values()) { + eachStorageManager.closeStorageManager(); + } + } + } + public List getSplits(String fragmentId, TableDesc tableDesc) throws IOException { return getSplits(fragmentId, tableDesc, null); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index 87faf3b917..1bfa875878 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -21,19 +21,18 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.InclusiveStopFilter; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.Scanner; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Bytes; @@ -51,7 +50,7 @@ public class HBaseScanner implements Scanner { private TableMeta meta; private HBaseFragment fragment; private Scan scan; - private HTable htable; + private HTableInterface htable; private Configuration hbaseConf; private Column[] targets; private TableStats tableStats; @@ -140,7 +139,11 @@ private void initScanner() throws IOException { } } - htable = new HTable(hbaseConf, fragment.getHbaseTableName()); + if (htable == null) { + HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE)) + .getConnection(hbaseConf); + htable = hconn.getTable(fragment.getHbaseTableName()); + } scanner = htable.getScanner(scan); } @@ -175,6 +178,7 @@ public void reset() throws IOException { if (scanner != null) { scanner.close(); + scanner = null; } initScanner(); @@ -187,12 +191,14 @@ public void close() throws IOException { if (scanner != null) { try { scanner.close(); + scanner = null; } catch (Exception e) { LOG.warn("Error while closing hbase scanner: " + e.getMessage(), e); } } if (htable != null) { htable.close(); + htable = null; } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 7de0cabd7d..94a792e900 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -24,8 +24,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.UserProvider; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; @@ -45,7 +46,7 @@ import java.util.*; public class HBaseStorageManager extends StorageManager { - private final Log LOG = LogFactory.getLog(HBaseStorageManager.class); + private static final Log LOG = LogFactory.getLog(HBaseStorageManager.class); public static final String META_TABLE_KEY = "table"; public static final String META_COLUMNS_KEY = "columns"; @@ -55,10 +56,25 @@ public class HBaseStorageManager extends StorageManager { public static final String ROWKEY_COLUMN_MAPPING = "key"; public static final String META_ROWKEY_DELIMITER = "hbase.rowkey.delimiter"; + private Map connMap = new HashMap(); + @Override public void storageInit() throws IOException { } + @Override + public void closeStorageManager() { + synchronized (connMap) { + for (HConnection eachConn: connMap.values()) { + try { + eachConn.close(); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + } + } + @Override public void createTable(TableDesc tableDesc) throws IOException { TableMeta tableMeta = tableDesc.getMeta(); @@ -73,51 +89,57 @@ public void createTable(TableDesc tableDesc) throws IOException { if (columnMapping != null && columnMapping.split(",").length > tableDesc.getSchema().size()) { throw new IOException("Columns property has more entry than Tajo table columns"); } - HBaseAdmin hAdmin = new HBaseAdmin(getHBaseConfiguration(conf, tableMeta)); - if (tableDesc.isExternal()) { - // If tajo table is external table, only check validation. - if (columnMapping == null || columnMapping.isEmpty()) { - throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); - } - if (!hAdmin.tableExists(hTableName)) { - throw new IOException ("HBase table [" + hbaseTableName + "] not exists. " + - "External table should be a existed table."); - } - HTableDescriptor hTableDescriptor = hAdmin.getTableDescriptor(hTableName); - Set tableColumnFamilies = new HashSet(); - for (HColumnDescriptor eachColumn: hTableDescriptor.getColumnFamilies()) { - tableColumnFamilies.add(eachColumn.getNameAsString()); - } + Configuration hConf = getHBaseConfiguration(conf, tableMeta); + HBaseAdmin hAdmin = new HBaseAdmin(hConf); - Collection mappingColumnFamilies = getColumnFamilies(columnMapping); - if (mappingColumnFamilies.isEmpty()) { - throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); - } + try { + if (tableDesc.isExternal()) { + // If tajo table is external table, only check validation. + if (columnMapping == null || columnMapping.isEmpty()) { + throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); + } + if (!hAdmin.tableExists(hTableName)) { + throw new IOException("HBase table [" + hbaseTableName + "] not exists. " + + "External table should be a existed table."); + } + HTableDescriptor hTableDescriptor = hAdmin.getTableDescriptor(hTableName); + Set tableColumnFamilies = new HashSet(); + for (HColumnDescriptor eachColumn : hTableDescriptor.getColumnFamilies()) { + tableColumnFamilies.add(eachColumn.getNameAsString()); + } - for (String eachMappingColumnFamily: mappingColumnFamilies) { - if (!tableColumnFamilies.contains(eachMappingColumnFamily)) { - throw new IOException ("There is no " + eachMappingColumnFamily + " column family in " + hbaseTableName); + Collection mappingColumnFamilies = getColumnFamilies(columnMapping); + if (mappingColumnFamilies.isEmpty()) { + throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); } - } - } else { - if (hAdmin.tableExists(hbaseTableName)) { - throw new IOException ("HBase table [" + hbaseTableName + "] already exists."); - } - // Creating hbase table - HTableDescriptor hTableDescriptor = parseHTableDescriptor(tableDesc); - byte[][] splitKeys = getSplitKeys(conf, tableMeta); - if (splitKeys == null) { - hAdmin.createTable(hTableDescriptor); + for (String eachMappingColumnFamily : mappingColumnFamilies) { + if (!tableColumnFamilies.contains(eachMappingColumnFamily)) { + throw new IOException("There is no " + eachMappingColumnFamily + " column family in " + hbaseTableName); + } + } } else { - hAdmin.createTable(hTableDescriptor, splitKeys); + if (hAdmin.tableExists(hbaseTableName)) { + throw new IOException("HBase table [" + hbaseTableName + "] already exists."); + } + // Creating hbase table + HTableDescriptor hTableDescriptor = parseHTableDescriptor(tableDesc); + + byte[][] splitKeys = getSplitKeys(conf, tableMeta); + if (splitKeys == null) { + hAdmin.createTable(hTableDescriptor); + } else { + hAdmin.createTable(hTableDescriptor, splitKeys); + } } - } - TableStats stats = new TableStats(); - stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); - tableDesc.setStats(stats); + TableStats stats = new TableStats(); + stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); + tableDesc.setStats(stats); + } finally { + hAdmin.close(); + } } private byte[][] getSplitKeys(TajoConf conf, TableMeta meta) throws IOException { @@ -251,11 +273,15 @@ public static HTableDescriptor parseHTableDescriptor(TableDesc tableDesc) throws @Override public void purgeTable(TableDesc tableDesc) throws IOException { - HBaseAdmin hAdmin = new HBaseAdmin(getHBaseConfiguration(conf, tableDesc.getMeta())); - - HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc); - hAdmin.disableTable(hTableDesc.getName()); - hAdmin.deleteTable(hTableDesc.getName()); + HBaseAdmin hAdmin = new HBaseAdmin(getHBaseConfiguration(conf, tableDesc.getMeta())); + + try { + HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc); + hAdmin.disableTable(hTableDesc.getName()); + hAdmin.deleteTable(hTableDesc.getName()); + } finally { + hAdmin.close(); + } } @Override @@ -280,91 +306,103 @@ public Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { public List getSplits(String fragmentId, TableDesc tableDesc, List indexPredications) throws IOException { Configuration hconf = getHBaseConfiguration(conf, tableDesc.getMeta()); - HTable htable = new HTable(hconf, tableDesc.getMeta().getOption(META_TABLE_KEY)); - - org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); - if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { - HRegionLocation regLoc = htable.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false); - if (null == regLoc) { - throw new IOException("Expecting at least one region."); - } - List fragments = new ArrayList(1); - Fragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), - HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc.getHostname()); - fragments.add(fragment); - return fragments; - } + HTable htable = null; + HBaseAdmin hAdmin = null; - ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); - byte[] startRow = HConstants.EMPTY_START_ROW; - byte[] stopRow = HConstants.EMPTY_END_ROW; - IndexPredication indexPredication = null; - if (indexPredications != null && !indexPredications.isEmpty()) { - // Currently only supports rowkey - indexPredication = indexPredications.get(0); + try { + htable = new HTable(hconf, tableDesc.getMeta().getOption(META_TABLE_KEY)); - if (indexPredication.getStartValue() != null) { - startRow = serialize(columnMapping, indexPredication, indexPredication.getStartValue()); - } - if (indexPredication.getStopValue() != null) { - stopRow = serialize(columnMapping, indexPredication, indexPredication.getStopValue()); + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { + HRegionLocation regLoc = htable.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false); + if (null == regLoc) { + throw new IOException("Expecting at least one region."); + } + List fragments = new ArrayList(1); + Fragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), + HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc.getHostname()); + fragments.add(fragment); + return fragments; } - } - HBaseAdmin hAdmin = new HBaseAdmin(conf); - Map serverLoadMap = new HashMap(); + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); + byte[] startRow = HConstants.EMPTY_START_ROW; + byte[] stopRow = HConstants.EMPTY_END_ROW; + IndexPredication indexPredication = null; + if (indexPredications != null && !indexPredications.isEmpty()) { + // Currently only supports rowkey + indexPredication = indexPredications.get(0); - List fragments = new ArrayList(keys.getFirst().length); - for (int i = 0; i < keys.getFirst().length; i++) { - HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); + if (indexPredication.getStartValue() != null) { + startRow = serialize(columnMapping, indexPredication, indexPredication.getStartValue()); + } + if (indexPredication.getStopValue() != null) { + stopRow = serialize(columnMapping, indexPredication, indexPredication.getStopValue()); + } + } - // determine if the given start an stop key fall into the region - if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) - && (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) { - byte[] fragmentStart = (startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0) ? - keys.getFirst()[i] : startRow; + hAdmin = new HBaseAdmin(hconf); + Map serverLoadMap = new HashMap(); - byte[] fragmentStop = (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && - keys.getSecond()[i].length > 0 ? keys.getSecond()[i] : stopRow; + List fragments = new ArrayList(keys.getFirst().length); + for (int i = 0; i < keys.getFirst().length; i++) { + HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); - String regionName = location.getRegionInfo().getRegionNameAsString(); + // determine if the given start an stop key fall into the region + if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) + && (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) { + byte[] fragmentStart = (startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0) ? + keys.getFirst()[i] : startRow; - ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); - if (serverLoad == null) { - serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); - serverLoadMap.put(location.getServerName(), serverLoad); - } + byte[] fragmentStop = (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && + keys.getSecond()[i].length > 0 ? keys.getSecond()[i] : stopRow; - HBaseFragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), - fragmentStart, fragmentStop, location.getHostname()); + String regionName = location.getRegionInfo().getRegionNameAsString(); - // get region size - boolean foundLength = false; - for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { - if (regionName.equals(Bytes.toString(entry.getKey()))) { - RegionLoad regionLoad = entry.getValue(); - long storeFileSize = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; - fragment.setLength(storeFileSize); - foundLength = true; - break; + ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); + if (serverLoad == null) { + serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); + serverLoadMap.put(location.getServerName(), serverLoad); } - } - if (!foundLength) { - fragment.setLength(TajoConstants.UNKNOWN_LENGTH); - } + HBaseFragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), + fragmentStart, fragmentStop, location.getHostname()); + + // get region size + boolean foundLength = false; + for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { + if (regionName.equals(Bytes.toString(entry.getKey()))) { + RegionLoad regionLoad = entry.getValue(); + long storeFileSize = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; + fragment.setLength(storeFileSize); + foundLength = true; + break; + } + } - fragments.add(fragment); - if (LOG.isDebugEnabled()) { - LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + if (!foundLength) { + fragment.setLength(TajoConstants.UNKNOWN_LENGTH); + } + + fragments.add(fragment); + if (LOG.isDebugEnabled()) { + LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + } } } - } - if (!fragments.isEmpty()) { - ((HBaseFragment)fragments.get(fragments.size() - 1)).setLast(true); + if (!fragments.isEmpty()) { + ((HBaseFragment)fragments.get(fragments.size() - 1)).setLast(true); + } + return fragments; + } finally { + if (htable != null) { + htable.close(); + } + if (hAdmin != null) { + hAdmin.close(); + } } - return fragments; } public static RowKeyMapping getRowKeyMapping(String cfName, String columnName) { @@ -402,73 +440,197 @@ private byte[] serialize(ColumnMapping columnMapping, public List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { Configuration hconf = getHBaseConfiguration(conf, tableDesc.getMeta()); - HTable htable = new HTable(hconf, tableDesc.getMeta().getOption(META_TABLE_KEY)); + HTable htable = null; + HBaseAdmin hAdmin = null; + try { + htable = new HTable(hconf, tableDesc.getMeta().getOption(META_TABLE_KEY)); + + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { + return new ArrayList(1); + } + hAdmin = new HBaseAdmin(hconf); + Map serverLoadMap = new HashMap(); - org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); - if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { - return new ArrayList(1); - } + List fragments = new ArrayList(keys.getFirst().length); - HBaseAdmin hAdmin = new HBaseAdmin(conf); - Map serverLoadMap = new HashMap(); + int start = currentPage * numFragments; + if (start >= keys.getFirst().length) { + return new ArrayList(1); + } + int end = (currentPage + 1) * numFragments; + if (end > keys.getFirst().length) { + end = keys.getFirst().length; + } + for (int i = start; i < end; i++) { + HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); - List fragments = new ArrayList(keys.getFirst().length); + String regionName = location.getRegionInfo().getRegionNameAsString(); + ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); + if (serverLoad == null) { + serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); + serverLoadMap.put(location.getServerName(), serverLoad); + } - int start = currentPage * numFragments; - if (start >= keys.getFirst().length) { - return new ArrayList(1); + HBaseFragment fragment = new HBaseFragment(tableDesc.getName(), htable.getName().getNameAsString(), + location.getRegionInfo().getStartKey(), location.getRegionInfo().getEndKey(), location.getHostname()); + + // get region size + boolean foundLength = false; + for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { + if (regionName.equals(Bytes.toString(entry.getKey()))) { + RegionLoad regionLoad = entry.getValue(); + long storeLength = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; + if (storeLength == 0) { + // If store size is smaller than 1 MB, storeLength is zero + storeLength = 1 * 1024 * 1024; //default 1MB + } + fragment.setLength(storeLength); + foundLength = true; + break; + } + } + + if (!foundLength) { + fragment.setLength(TajoConstants.UNKNOWN_LENGTH); + } + + fragments.add(fragment); + if (LOG.isDebugEnabled()) { + LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + } + } + + if (!fragments.isEmpty()) { + ((HBaseFragment) fragments.get(fragments.size() - 1)).setLast(true); + } + return fragments; + } finally { + if (htable != null) { + htable.close(); + } + if (hAdmin != null) { + hAdmin.close(); + } } - int end = (currentPage + 1) * numFragments; - if (end > keys.getFirst().length) { - end = keys.getFirst().length; + } + + @Override + public boolean canCreateAsSelect(StoreType storeType) { + return false; + } + + public HConnection getConnection(Configuration hbaseConf) throws IOException { + synchronized(connMap) { + HConnectionKey key = new HConnectionKey(hbaseConf); + HConnection conn = connMap.get(key); + if (conn == null) { + conn = HConnectionManager.createConnection(hbaseConf); + connMap.put(key, conn); + } + + return conn; } - for (int i = start; i < end; i++) { - HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); - - String regionName = location.getRegionInfo().getRegionNameAsString(); - ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); - if (serverLoad == null) { - serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); - serverLoadMap.put(location.getServerName(), serverLoad); - } - - HBaseFragment fragment = new HBaseFragment(tableDesc.getName(), htable.getName().getNameAsString(), - location.getRegionInfo().getStartKey(), location.getRegionInfo().getEndKey(), location.getHostname()); - - // get region size - boolean foundLength = false; - for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { - if (regionName.equals(Bytes.toString(entry.getKey()))) { - RegionLoad regionLoad = entry.getValue(); - long storeLength = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; - if (storeLength == 0) { - // If store size is smaller than 1 MB, storeLength is zero - storeLength = 1 * 1024 * 1024; //default 1MB + } + + static class HConnectionKey { + final static String[] CONNECTION_PROPERTIES = new String[] { + HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.ZOOKEEPER_CLIENT_PORT, + HConstants.ZOOKEEPER_RECOVERABLE_WAITTIME, + HConstants.HBASE_CLIENT_PAUSE, HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.HBASE_RPC_TIMEOUT_KEY, + HConstants.HBASE_CLIENT_PREFETCH_LIMIT, + HConstants.HBASE_META_SCANNER_CACHING, + HConstants.HBASE_CLIENT_INSTANCE_ID, + HConstants.RPC_CODEC_CONF_KEY }; + + private Map properties; + private String username; + + HConnectionKey(Configuration conf) { + Map m = new HashMap(); + if (conf != null) { + for (String property : CONNECTION_PROPERTIES) { + String value = conf.get(property); + if (value != null) { + m.put(property, value); } - fragment.setLength(storeLength); - foundLength = true; - break; } } + this.properties = Collections.unmodifiableMap(m); - if (!foundLength) { - fragment.setLength(TajoConstants.UNKNOWN_LENGTH); + try { + UserProvider provider = UserProvider.instantiate(conf); + User currentUser = provider.getCurrent(); + if (currentUser != null) { + username = currentUser.getName(); + } + } catch (IOException ioe) { + LOG.warn("Error obtaining current user, skipping username in HConnectionKey", ioe); } + } - fragments.add(fragment); - if (LOG.isDebugEnabled()) { - LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + if (username != null) { + result = username.hashCode(); + } + for (String property : CONNECTION_PROPERTIES) { + String value = properties.get(property); + if (value != null) { + result = prime * result + value.hashCode(); + } } + + return result; } - if (!fragments.isEmpty()) { - ((HBaseFragment)fragments.get(fragments.size() - 1)).setLast(true); + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + HConnectionKey that = (HConnectionKey) obj; + if (this.username != null && !this.username.equals(that.username)) { + return false; + } else if (this.username == null && that.username != null) { + return false; + } + if (this.properties == null) { + if (that.properties != null) { + return false; + } + } else { + if (that.properties == null) { + return false; + } + for (String property : CONNECTION_PROPERTIES) { + String thisValue = this.properties.get(property); + String thatValue = that.properties.get(property); + //noinspection StringEquality + if (thisValue == thatValue) { + continue; + } + if (thisValue == null || !thisValue.equals(thatValue)) { + return false; + } + } + } + return true; } - return fragments; - } - @Override - public boolean canCreateAsSelect(StoreType storeType) { - return false; + @Override + public String toString() { + return "HConnectionKey{" + + "properties=" + properties + + ", username='" + username + '\'' + + '}'; + } } } From 77941b843c99786e4eafd26b8d4d32f5349c38aa Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 27 Oct 2014 15:20:34 +0900 Subject: [PATCH 05/29] TAJO-1127: Implements HBaseStorageManager. --- .../master/querymaster/Repartitioner.java | 5 +- .../tajo/master/querymaster/SubQuery.java | 2 +- .../tajo/engine/query/TestHBaseTable.java | 69 ++++- .../tajo/storage/FileStorageManager.java | 10 +- .../apache/tajo/storage/StorageManager.java | 4 +- .../tajo/storage/hbase/HBaseFragment.java | 8 + .../storage/hbase/HBaseStorageManager.java | 272 ++++++++++-------- .../storage/{ => hbase}/IndexPredication.java | 2 +- .../tajo/storage/TestHBaseStorageManager.java | 111 +++++++ 9 files changed, 348 insertions(+), 135 deletions(-) rename tajo-storage/src/main/java/org/apache/tajo/storage/{ => hbase}/IndexPredication.java (97%) create mode 100644 tajo-storage/src/test/java/org/apache/tajo/storage/TestHBaseStorageManager.java diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 046b310ae8..c9ea32acac 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -419,7 +419,8 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster } else { StorageManager storageManager = StorageManager.getStorageManager(subQuery.getContext().getConf(), tableDesc.getMeta().getStoreType()); - Collection scanFragments = storageManager.getSplits(eachScan.getCanonicalName(), tableDesc); + Collection scanFragments = storageManager.getSplits(eachScan.getCanonicalName(), + tableDesc, eachScan); if (scanFragments != null) { rightFragments.addAll(scanFragments); } @@ -539,7 +540,7 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch StorageManager storageManager = StorageManager.getStorageManager(subQuery.getContext().getConf(), desc.getMeta().getStoreType()); - scanFragments = storageManager.getSplits(scan.getCanonicalName(), desc); + scanFragments = storageManager.getSplits(scan.getCanonicalName(), desc, scan); } if (scanFragments != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index ec9afaa842..2a489f8b0d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -985,7 +985,7 @@ private static void scheduleFragmentsForLeafQuery(SubQuery subQuery) throws IOEx } else { StorageManager storageManager = StorageManager.getStorageManager(subQuery.getContext().getConf(), meta.getStoreType()); - fragments = storageManager.getSplits(scan.getCanonicalName(), table); + fragments = storageManager.getSplits(scan.getCanonicalName(), table, scan); } SubQuery.scheduleFragments(subQuery, fragments); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index d1ef23f05d..368b5d479f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -29,7 +29,6 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.hbase.HBaseFragment; @@ -44,8 +43,10 @@ import java.sql.ResultSet; import java.text.DecimalFormat; import java.util.List; +import java.util.Set; import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; @Category(IntegrationTest.class) public class TestHBaseTable extends QueryTestCaseBase { @@ -350,8 +351,7 @@ public void testIndexPredication() throws Exception { scanNode.setQual(evalNodeA); StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE); - List fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, - HBaseStorageManager.getIndexPredications(storageManager, tableDesc, scanNode)); + List fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); assertEquals(2, fragments.size()); HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); @@ -362,6 +362,69 @@ public void testIndexPredication() throws Exception { assertEquals("040", new String(fragment2.getStartRow())); assertEquals("055", new String(fragment2.getStopRow())); + + // where (rk >= '020' and rk <= '055') or rk = '075' + EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("075"))); + EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); + scanNode.setQual(evalNodeB); + fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); + assertEquals(3, fragments.size()); + fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("055", new String(fragment2.getStopRow())); + + HBaseFragment fragment3 = (HBaseFragment) fragments.get(2); + assertEquals("075", new String(fragment3.getStartRow())); + assertEquals("075", new String(fragment3.getStopRow())); + + + // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078') + EvalNode evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("072"))); + EvalNode evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("078"))); + EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); + EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); + scanNode.setQual(evalNodeD); + fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); + assertEquals(3, fragments.size()); + + fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("055", new String(fragment2.getStopRow())); + + fragment3 = (HBaseFragment) fragments.get(2); + assertEquals("072", new String(fragment3.getStartRow())); + assertEquals("078", new String(fragment3.getStopRow())); + + // where (rk >= '020' and rk <= '055') or (rk >= '057' and rk <= '059') + evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("057"))); + evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("059"))); + evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); + evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); + scanNode.setQual(evalNodeD); + fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); + assertEquals(2, fragments.size()); + + fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("059", new String(fragment2.getStopRow())); + ResultSet res = executeString("select * from external_hbase_mapped_table where rk >= '020' and rk <= '055'"); assertResultSet(res); res.close(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 8815a74bbc..cef7f48a9d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -29,20 +29,17 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.fragment.FragmentConvertor; +import org.apache.tajo.storage.hbase.IndexPredication; import org.apache.tajo.util.*; import java.io.FileNotFoundException; import java.io.IOException; -import java.lang.reflect.Constructor; -import java.net.URI; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; @@ -638,8 +635,7 @@ public String getMessage(){ } @Override - public List getSplits(String tableName, TableDesc table, - List indexPredications) throws IOException { + public List getSplits(String tableName, TableDesc table, ScanNode scanNode) throws IOException { return getSplits(tableName, table.getMeta(), table.getSchema(), table.getPath()); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index a0b5957b3b..4280c9cb90 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -36,6 +36,8 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -83,7 +85,7 @@ public abstract class StorageManager { public abstract void createTable(TableDesc tableDesc) throws IOException; public abstract void purgeTable(TableDesc tableDesc) throws IOException; public abstract List getSplits(String fragmentId, TableDesc tableDesc, - List indexPredications) throws IOException; + ScanNode scanNode) throws IOException; public abstract List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException; public abstract Column[] getIndexableColumns(TableDesc tableDesc) throws IOException; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java index 52c5e7abc1..43ad7f3aed 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -187,4 +187,12 @@ public String getHbaseTableName() { public void setHbaseTableName(String hbaseTableName) { this.hbaseTableName = hbaseTableName; } + + public void setStartRow(byte[] startRow) { + this.startRow = startRow; + } + + public void setStopRow(byte[] stopRow) { + this.stopRow = stopRow; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 7e05e1afff..04b941d4ea 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -38,12 +38,11 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.IndexPredication; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Bytes; import org.apache.tajo.util.Pair; +import org.apache.tajo.util.TUtil; import java.io.BufferedReader; import java.io.IOException; @@ -308,8 +307,8 @@ public Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { } @Override - public List getSplits(String fragmentId, TableDesc tableDesc, - List indexPredications) throws IOException { + public List getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException { + List indexPredications = getIndexPredications(tableDesc, scanNode); Configuration hconf = getHBaseConfiguration(conf, tableDesc.getMeta()); HTable htable = null; HBaseAdmin hAdmin = null; @@ -331,75 +330,109 @@ public List getSplits(String fragmentId, TableDesc tableDesc, } ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); - byte[] startRow = HConstants.EMPTY_START_ROW; - byte[] stopRow = HConstants.EMPTY_END_ROW; - IndexPredication indexPredication = null; - if (indexPredications != null && !indexPredications.isEmpty()) { - // Currently only supports rowkey - indexPredication = indexPredications.get(0); + List startRows; + List stopRows; - if (indexPredication.getStartValue() != null) { - startRow = serialize(columnMapping, indexPredication, indexPredication.getStartValue()); - } - if (indexPredication.getStopValue() != null) { - stopRow = serialize(columnMapping, indexPredication, indexPredication.getStopValue()); + if (indexPredications != null && !indexPredications.isEmpty()) { + // indexPredications is Disjunctive set + startRows = new ArrayList(); + stopRows = new ArrayList(); + for (IndexPredication indexPredication: indexPredications) { + byte[] startRow; + byte[] stopRow; + if (indexPredication.getStartValue() != null) { + startRow = serialize(columnMapping, indexPredication, indexPredication.getStartValue()); + } else { + startRow = HConstants.EMPTY_START_ROW; + } + if (indexPredication.getStopValue() != null) { + stopRow = serialize(columnMapping, indexPredication, indexPredication.getStopValue()); + } else { + stopRow = HConstants.EMPTY_END_ROW; + } + startRows.add(startRow); + stopRows.add(stopRow); } + } else { + startRows = TUtil.newList(HConstants.EMPTY_START_ROW); + stopRows = TUtil.newList(HConstants.EMPTY_END_ROW); } hAdmin = new HBaseAdmin(hconf); Map serverLoadMap = new HashMap(); - List fragments = new ArrayList(keys.getFirst().length); + // region startkey -> HBaseFragment + Map fragmentMap = new HashMap(); for (int i = 0; i < keys.getFirst().length; i++) { HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); - // determine if the given start an stop key fall into the region - if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) - && (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) { - byte[] fragmentStart = (startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0) ? - keys.getFirst()[i] : startRow; + byte[] regionStartKey = keys.getFirst()[i]; + byte[] regionStopKey = keys.getSecond()[i]; - byte[] fragmentStop = (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && - keys.getSecond()[i].length > 0 ? keys.getSecond()[i] : stopRow; + int startRowsSize = startRows.size(); + for (int j = 0; j < startRowsSize; j++) { + byte[] startRow = startRows.get(j); + byte[] stopRow = stopRows.get(j); + // determine if the given start an stop key fall into the region + if ((startRow.length == 0 || regionStopKey.length == 0 || Bytes.compareTo(startRow, regionStopKey) < 0) + && (stopRow.length == 0 || Bytes.compareTo(stopRow, regionStartKey) > 0)) { + byte[] fragmentStart = (startRow.length == 0 || Bytes.compareTo(regionStartKey, startRow) >= 0) ? + regionStartKey : startRow; - String regionName = location.getRegionInfo().getRegionNameAsString(); + byte[] fragmentStop = (stopRow.length == 0 || Bytes.compareTo(regionStopKey, stopRow) <= 0) && + regionStopKey.length > 0 ? regionStopKey : stopRow; - ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); - if (serverLoad == null) { - serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); - serverLoadMap.put(location.getServerName(), serverLoad); - } + String regionName = location.getRegionInfo().getRegionNameAsString(); - HBaseFragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), - fragmentStart, fragmentStop, location.getHostname()); - - // get region size - boolean foundLength = false; - for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { - if (regionName.equals(Bytes.toString(entry.getKey()))) { - RegionLoad regionLoad = entry.getValue(); - long storeFileSize = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; - fragment.setLength(storeFileSize); - foundLength = true; - break; + ServerLoad serverLoad = serverLoadMap.get(location.getServerName()); + if (serverLoad == null) { + serverLoad = hAdmin.getClusterStatus().getLoad(location.getServerName()); + serverLoadMap.put(location.getServerName(), serverLoad); } - } - - if (!foundLength) { - fragment.setLength(TajoConstants.UNKNOWN_LENGTH); - } - fragments.add(fragment); - if (LOG.isDebugEnabled()) { - LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + if (fragmentMap.containsKey(regionStartKey)) { + HBaseFragment prevFragment = fragmentMap.get(regionStartKey); + if (Bytes.compareTo(fragmentStart, prevFragment.getStartRow()) < 0) { + prevFragment.setStartRow(fragmentStart); + } + if (Bytes.compareTo(fragmentStop, prevFragment.getStopRow()) > 0) { + prevFragment.setStopRow(fragmentStop); + } + } else { + HBaseFragment fragment = new HBaseFragment(fragmentId, htable.getName().getNameAsString(), + fragmentStart, fragmentStop, location.getHostname()); + + // get region size + boolean foundLength = false; + for (Map.Entry entry : serverLoad.getRegionsLoad().entrySet()) { + if (regionName.equals(Bytes.toString(entry.getKey()))) { + RegionLoad regionLoad = entry.getValue(); + long storeFileSize = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * 1024L * 1024L; + fragment.setLength(storeFileSize); + foundLength = true; + break; + } + } + + if (!foundLength) { + fragment.setLength(TajoConstants.UNKNOWN_LENGTH); + } + + fragmentMap.put(regionStartKey, fragment); + if (LOG.isDebugEnabled()) { + LOG.debug("getFragments: fragment -> " + i + " -> " + fragment); + } + } } } } + List fragments = new ArrayList(fragmentMap.values()); + Collections.sort(fragments); if (!fragments.isEmpty()) { - ((HBaseFragment)fragments.get(fragments.size() - 1)).setLast(true); + fragments.get(fragments.size() - 1).setLast(true); } - return fragments; + return (ArrayList) (ArrayList) fragments; } finally { if (htable != null) { htable.close(); @@ -639,63 +672,57 @@ public String toString() { } } - public static List getIndexPredications(StorageManager storageHandler, - TableDesc tableDesc, ScanNode scan) throws IOException { + public List getIndexPredications(TableDesc tableDesc, ScanNode scanNode) throws IOException { List indexPredications = new ArrayList(); - Column[] indexableColumns = storageHandler.getIndexableColumns(tableDesc); + Column[] indexableColumns = getIndexableColumns(tableDesc); if (indexableColumns != null && indexableColumns.length == 1) { // Currently supports only single index column. - Set indexablePredicateSet = findIndexablePredicateSet(scan, indexableColumns, true); - Pair indexPredicationValues = getIndexablePredicateValue(indexablePredicateSet); - if (indexPredicationValues != null) { - IndexPredication indexPredication = new IndexPredication(); - indexPredication.setColumn(indexableColumns[0]); - indexPredication.setColumnId(tableDesc.getLogicalSchema().getColumnId(indexableColumns[0].getQualifiedName())); - indexPredication.setStartValue(indexPredicationValues.getFirst()); - indexPredication.setStopValue(indexPredicationValues.getSecond()); + List> indexablePredicateList = findIndexablePredicateSet(scanNode, indexableColumns); + for (Set eachEvalSet: indexablePredicateList) { + Pair indexPredicationValues = getIndexablePredicateValue(eachEvalSet); + if (indexPredicationValues != null) { + IndexPredication indexPredication = new IndexPredication(); + indexPredication.setColumn(indexableColumns[0]); + indexPredication.setColumnId(tableDesc.getLogicalSchema().getColumnId(indexableColumns[0].getQualifiedName())); + indexPredication.setStartValue(indexPredicationValues.getFirst()); + indexPredication.setStopValue(indexPredicationValues.getSecond()); - indexPredications.add(indexPredication); + indexPredications.add(indexPredication); + } } } return indexPredications; } - public static Set findIndexablePredicateSet(ScanNode scanNode, Column[] indexableColumns, - boolean preserveFoundPredicate) throws IOException { - Set indexablePredicateSet = Sets.newHashSet(); + public List> findIndexablePredicateSet(ScanNode scanNode, Column[] indexableColumns) throws IOException { + List> indexablePredicateList = new ArrayList>(); + // if a query statement has a search condition, try to find indexable predicates - if (indexableColumns != null && scanNode.hasQual()) { - EvalNode[] conjunctiveForms = AlgebraicUtil.toConjunctiveNormalFormArray(scanNode.getQual()); - Set remainExprs = Sets.newHashSet(conjunctiveForms); + if (indexableColumns != null && scanNode.getQual() != null) { + EvalNode[] disjunctiveForms = AlgebraicUtil.toDisjunctiveNormalFormArray(scanNode.getQual()); // add qualifier to schema for qual for (Column column : indexableColumns) { - for (EvalNode simpleExpr : conjunctiveForms) { - if (checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) { - indexablePredicateSet.add(simpleExpr); + for (EvalNode disjunctiveExpr : disjunctiveForms) { + EvalNode[] conjunctiveForms = AlgebraicUtil.toConjunctiveNormalFormArray(disjunctiveExpr); + Set indexablePredicateSet = Sets.newHashSet(); + for (EvalNode conjunctiveExpr : conjunctiveForms) { + if (checkIfIndexablePredicateOnTargetColumn(conjunctiveExpr, column)) { + indexablePredicateSet.add(conjunctiveExpr); + } + } + if (!indexablePredicateSet.isEmpty()) { + indexablePredicateList.add(indexablePredicateSet); } - } - } - - // Partitions which are not matched to the partition filter conditions are pruned immediately. - // So, the partition filter conditions are not necessary later, and they are removed from - // original search condition for simplicity and efficiency. - remainExprs.removeAll(indexablePredicateSet); - if (!preserveFoundPredicate) { - if (remainExprs.isEmpty()) { - scanNode.setQual(null); - } else { - scanNode.setQual( - AlgebraicUtil.createSingletonExprFromCNF(remainExprs.toArray(new EvalNode[remainExprs.size()]))); } } } - return indexablePredicateSet; + return indexablePredicateList; } - public static boolean checkIfIndexablePredicateOnTargetColumn(EvalNode evalNode, Column targetColumn) { - if (checkIfIndexablePredicate(evalNode) || checkIfDisjunctiveButOneVariable(evalNode)) { + private boolean checkIfIndexablePredicateOnTargetColumn(EvalNode evalNode, Column targetColumn) { + if (checkIfIndexablePredicate(evalNode) || checkIfConjunctiveButOneVariable(evalNode)) { Set variables = EvalTreeUtil.findUniqueColumns(evalNode); // if it contains only single variable matched to a target column return variables.size() == 1 && variables.contains(targetColumn); @@ -707,10 +734,10 @@ public static boolean checkIfIndexablePredicateOnTargetColumn(EvalNode evalNode, /** * * @param evalNode The expression to be checked - * @return true if an disjunctive expression, consisting of indexable expressions + * @return true if an conjunctive expression, consisting of indexable expressions */ - public static boolean checkIfDisjunctiveButOneVariable(EvalNode evalNode) { - if (evalNode.getType() == EvalType.OR) { + private boolean checkIfConjunctiveButOneVariable(EvalNode evalNode) { + if (evalNode.getType() == EvalType.AND) { BinaryEval orEval = (BinaryEval) evalNode; boolean indexable = checkIfIndexablePredicate(orEval.getLeftExpr()) && @@ -734,19 +761,23 @@ public static boolean checkIfDisjunctiveButOneVariable(EvalNode evalNode) { * @return true if an expression consists of one variable and one constant * and the expression is a comparison operator. Other, false. */ - public static boolean checkIfIndexablePredicate(EvalNode evalNode) { - // TODO - LIKE with a trailing wild-card character and IN with an array can be indexable - return AlgebraicUtil.containSingleVar(evalNode) && AlgebraicUtil.isIndexableOperator(evalNode); + private boolean checkIfIndexablePredicate(EvalNode evalNode) { + return AlgebraicUtil.containSingleVar(evalNode) && isIndexableOperator(evalNode); } - public static Pair getIndexablePredicateValue(Set indexablePredicateSet) { - if (indexablePredicateSet.size() > 2) { - return null; - } + public static boolean isIndexableOperator(EvalNode expr) { + return expr.getType() == EvalType.EQUAL || + expr.getType() == EvalType.LEQ || + expr.getType() == EvalType.LTH || + expr.getType() == EvalType.GEQ || + expr.getType() == EvalType.GTH || + expr.getType() == EvalType.BETWEEN; + } + public Pair getIndexablePredicateValue(Set evalNodes) { Datum startDatum = null; Datum endDatum = null; - for (EvalNode evalNode: indexablePredicateSet) { + for (EvalNode evalNode: evalNodes) { if (evalNode instanceof BinaryEval) { BinaryEval binaryEval = (BinaryEval) evalNode; EvalNode left = binaryEval.getLeftExpr(); @@ -759,45 +790,46 @@ public static Pair getIndexablePredicateValue(Set indexa constValue = ((ConstEval) right).getValue(); } - if (evalNode.getType() == EvalType.EQUAL || - evalNode.getType() == EvalType.GEQ || - evalNode.getType() == EvalType.GTH) { - if (startDatum != null) { - if (constValue.compareTo(startDatum) < 0) { + if (constValue != null) { + if (evalNode.getType() == EvalType.EQUAL || + evalNode.getType() == EvalType.GEQ || + evalNode.getType() == EvalType.GTH) { + if (startDatum != null) { + if (constValue.compareTo(startDatum) > 0) { + startDatum = constValue; + } + } else { startDatum = constValue; } - } else { - startDatum = constValue; } - } - if (evalNode.getType() == EvalType.EQUAL || - evalNode.getType() == EvalType.LEQ || - evalNode.getType() == EvalType.LTH) { - if (endDatum != null) { - if (constValue.compareTo(endDatum) > 0) { + if (evalNode.getType() == EvalType.EQUAL || + evalNode.getType() == EvalType.LEQ || + evalNode.getType() == EvalType.LTH) { + if (endDatum != null) { + if (constValue.compareTo(endDatum) < 0) { + endDatum = constValue; + } + } else { endDatum = constValue; } - } else { - endDatum = constValue; } } } else if (evalNode instanceof BetweenPredicateEval) { BetweenPredicateEval betweenEval = (BetweenPredicateEval) evalNode; - if (betweenEval.getBegin().getType() == EvalType.CONST) { + if (betweenEval.getBegin().getType() == EvalType.CONST && betweenEval.getEnd().getType() == EvalType.CONST) { Datum value = ((ConstEval) betweenEval.getBegin()).getValue(); if (startDatum != null) { - if (value.compareTo(startDatum) < 0) { + if (value.compareTo(startDatum) > 0) { startDatum = value; } } else { startDatum = value; } - } - if (betweenEval.getEnd().getType() == EvalType.CONST) { - Datum value = ((ConstEval) betweenEval.getEnd()).getValue(); + + value = ((ConstEval) betweenEval.getEnd()).getValue(); if (endDatum != null) { - if (value.compareTo(endDatum) > 0) { + if (value.compareTo(endDatum) < 0) { endDatum = value; } } else { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/IndexPredication.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java similarity index 97% rename from tajo-storage/src/main/java/org/apache/tajo/storage/IndexPredication.java rename to tajo-storage/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java index b3f9a487aa..3a58e5034c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/IndexPredication.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.storage; +package org.apache.tajo.storage.hbase; import org.apache.tajo.catalog.Column; import org.apache.tajo.datum.Datum; diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestHBaseStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestHBaseStorageManager.java new file mode 100644 index 0000000000..268dd70c13 --- /dev/null +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestHBaseStorageManager.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.storage.hbase.HBaseStorageManager; +import org.apache.tajo.util.Pair; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class TestHBaseStorageManager { + @Test + public void testGetIndexPredications() throws Exception { + Column rowkeyColumn = new Column("rk", Type.TEXT); + // where rk >= '020' and rk <= '055' + ScanNode scanNode = new ScanNode(1); + EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("020"))); + EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("055"))); + EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); + scanNode.setQual(evalNodeA); + + HBaseStorageManager storageManager = + (HBaseStorageManager)StorageManager.getStorageManager(new TajoConf(), StoreType.HBASE); + List> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + assertNotNull(indexEvals); + assertEquals(1, indexEvals.size()); + Pair indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + assertEquals("020", indexPredicateValue.getFirst().asChars()); + assertEquals("055", indexPredicateValue.getSecond().asChars()); + + // where (rk >= '020' and rk <= '055') or rk = '075' + EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(rowkeyColumn),new ConstEval(new TextDatum("075"))); + EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); + scanNode.setQual(evalNodeB); + indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + assertEquals(2, indexEvals.size()); + indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + assertEquals("020", indexPredicateValue.getFirst().asChars()); + assertEquals("055", indexPredicateValue.getSecond().asChars()); + + indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(1)); + assertEquals("075", indexPredicateValue.getFirst().asChars()); + assertEquals("075", indexPredicateValue.getSecond().asChars()); + + // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078') + EvalNode evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("072"))); + EvalNode evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("078"))); + EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); + EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); + scanNode.setQual(evalNodeD); + indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + assertEquals(2, indexEvals.size()); + + indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + assertEquals("020", indexPredicateValue.getFirst().asChars()); + assertEquals("055", indexPredicateValue.getSecond().asChars()); + + indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(1)); + assertEquals("072", indexPredicateValue.getFirst().asChars()); + assertEquals("078", indexPredicateValue.getSecond().asChars()); + + // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078' and rk >= '073') + evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("072"))); + evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("078"))); + evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); + EvalNode evalNode6 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("073"))); + evalNodeD = new BinaryEval(EvalType.AND, evalNodeC, evalNode6); + EvalNode evalNodeE = new BinaryEval(EvalType.OR, evalNodeA, evalNodeD); + scanNode.setQual(evalNodeE); + indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); + assertEquals(2, indexEvals.size()); + + indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + assertEquals("020", indexPredicateValue.getFirst().asChars()); + assertEquals("055", indexPredicateValue.getSecond().asChars()); + + indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(1)); + assertEquals("073", indexPredicateValue.getFirst().asChars()); + assertEquals("078", indexPredicateValue.getSecond().asChars()); + } +} From 40aa361cd480c8e29798d11e4e894fc491be661d Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Thu, 30 Oct 2014 20:08:19 +0900 Subject: [PATCH 06/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. --- .../main/java/org/apache/tajo/QueryVars.java | 45 ++ .../engine/planner/global/ExecutionBlock.java | 2 + .../physical/ColPartitionStoreExec.java | 4 +- .../planner/physical/ExternalSortExec.java | 4 +- .../planner/physical/StoreTableExec.java | 37 +- .../tajo/engine/query/QueryContext.java | 30 +- .../org/apache/tajo/master/GlobalEngine.java | 22 +- .../apache/tajo/master/querymaster/Query.java | 350 ++------------ .../master/querymaster/QueryMasterTask.java | 36 +- .../master/querymaster/Repartitioner.java | 68 ++- .../tajo/master/querymaster/SubQuery.java | 11 +- .../java/org/apache/tajo/worker/Task.java | 47 -- .../tajo/worker/TaskAttemptContext.java | 16 +- .../org/apache/tajo/QueryTestCaseBase.java | 2 +- .../planner/physical/TestPhysicalPlanner.java | 18 +- .../tajo/engine/query/TestHBaseTable.java | 268 ++++++++++- .../worker/TestRangeRetrieverHandler.java | 4 +- .../queries/TestHBaseTable/testCATS.sql | 4 - .../results/TestHBaseTable/testCATS.result | 100 ++++ .../TestHBaseTable/testInsertInto.result | 3 + .../testInsertIntoMultiRegion.result | 100 ++++ .../testInsertIntoRowField.result | 4 + .../apache/tajo/plan/LogicalOptimizer.java | 7 + .../apache/tajo/plan/logical/SortNode.java | 14 + .../rules/AddSortForInsertRewriter.java | 62 +++ .../apache/tajo/plan/util/PlannerUtil.java | 72 ++- .../plan/verifier/PreLogicalPlanVerifier.java | 5 +- tajo-storage/pom.xml | 11 + .../java/org/apache/tajo/storage/CSVFile.java | 8 +- .../org/apache/tajo/storage/FileAppender.java | 28 +- .../tajo/storage/FileStorageManager.java | 440 +++++++++++++++--- .../java/org/apache/tajo/storage/RawFile.java | 6 +- .../java/org/apache/tajo/storage/RowFile.java | 6 +- .../apache/tajo/storage/StorageManager.java | 156 +++++-- .../apache/tajo/storage/StorageProperty.java | 40 ++ .../tajo/storage/avro/AvroAppender.java | 8 +- .../tajo/storage/hbase/ColumnMapping.java | 7 + .../tajo/storage/hbase/HBaseAppender.java | 236 ++++++++++ .../HBaseBinarySerializerDeserializer.java | 23 +- .../tajo/storage/hbase/HBaseLazyTuple.java | 16 +- .../tajo/storage/hbase/HBaseScanner.java | 8 +- .../storage/hbase/HBaseStorageManager.java | 237 ++++++++-- .../HBaseTextSerializerDeserializer.java | 26 +- .../tajo/storage/parquet/ParquetAppender.java | 9 +- .../apache/tajo/storage/rcfile/RCFile.java | 6 +- .../sequencefile/SequenceFileAppender.java | 6 +- .../tajo/storage/trevni/TrevniAppender.java | 6 +- .../tajo/storage/hbase/TestHBaseAppender.java | 52 +++ .../{ => hbase}/TestHBaseStorageManager.java | 5 +- 49 files changed, 2001 insertions(+), 674 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/QueryVars.java delete mode 100644 tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/StorageProperty.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java create mode 100644 tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java rename tajo-storage/src/test/java/org/apache/tajo/storage/{ => hbase}/TestHBaseStorageManager.java (96%) diff --git a/tajo-common/src/main/java/org/apache/tajo/QueryVars.java b/tajo-common/src/main/java/org/apache/tajo/QueryVars.java new file mode 100644 index 0000000000..b6ba7556e0 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/QueryVars.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +public enum QueryVars implements ConfigKey { + COMMAND_TYPE, + STAGING_DIR, + OUTPUT_TABLE_NAME, + OUTPUT_TABLE_PATH, + OUTPUT_PARTITIONS, + OUTPUT_OVERWRITE, + OUTPUT_AS_DIRECTORY, + OUTPUT_PER_FILE_SIZE, + ; + + QueryVars() { + } + + @Override + public String keyname() { + return name().toLowerCase(); + } + + @Override + public ConfigType type() { + return ConfigType.QUERY; + } +} + diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java index 77eb32d29e..aecb36438b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java @@ -82,6 +82,8 @@ public void setPlan(LogicalNode plan) { } else if (node instanceof TableSubQueryNode) { TableSubQueryNode subQuery = (TableSubQueryNode) node; s.add(s.size(), subQuery.getSubQuery()); + } else if (node instanceof StoreTableNode) { + store = (StoreTableNode)node; } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 7818fd7c5a..1827461058 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -119,7 +119,9 @@ public ColPartitionStoreExec(TaskAttemptContext context, StoreTableNode plan, Ph public void init() throws IOException { super.init(); - storeTablePath = context.getOutputPath(); + storeTablePath = StorageManager.getFileStorageManager(context.getConf()) + .getAppenderFilePath(context.getTaskId(), context.getWorkDir()); + FileSystem fs = storeTablePath.getFileSystem(context.getConf()); if (!fs.exists(storeTablePath.getParent())) { fs.mkdirs(storeTablePath.getParent()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java index 105c374d26..536cbe6d28 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java @@ -173,7 +173,7 @@ private Path sortAndStoreChunk(int chunkId, List tupleBlock) long chunkWriteStart = System.currentTimeMillis(); Path outputPath = getChunkPathForWrite(0, chunkId); - final RawFileAppender appender = new RawFileAppender(context.getConf(), inSchema, meta, outputPath); + final RawFileAppender appender = new RawFileAppender(context.getConf(), null, inSchema, meta, outputPath); appender.init(); for (Tuple t : tupleBlock) { appender.addTuple(t); @@ -471,7 +471,7 @@ public FileFragment call() throws Exception { final Path outputPath = getChunkPathForWrite(level + 1, nextRunId); info(LOG, mergeFanout + " files are being merged to an output file " + outputPath.getName()); long mergeStartTime = System.currentTimeMillis(); - final RawFileAppender output = new RawFileAppender(context.getConf(), inSchema, meta, outputPath); + final RawFileAppender output = new RawFileAppender(context.getConf(), null, inSchema, meta, outputPath); output.init(); final Scanner merger = createKWayMerger(inputFiles, startIdx, mergeFanout); merger.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index 725478f77f..64d8c7be46 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -23,11 +23,13 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.plan.logical.InsertNode; import org.apache.tajo.plan.logical.PersistentStoreNode; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; @@ -78,31 +80,32 @@ public void init() throws IOException { } public void openNewFile(int suffixId) throws IOException { - String prevFile = null; + Schema appenderSchema = (plan instanceof InsertNode) ? ((InsertNode) plan).getTableSchema() : outSchema; - lastFileName = context.getOutputPath(); - if (suffixId > 0) { - prevFile = lastFileName.toString(); + if (PlannerUtil.isFileStorageType(meta.getStoreType())) { + String prevFile = null; - lastFileName = new Path(lastFileName + "_" + suffixId); - } + lastFileName = StorageManager.getFileStorageManager(context.getConf()).getAppenderFilePath( + context.getTaskId(), context.getQueryContext().getStagingDir()); + + if (suffixId > 0) { + prevFile = lastFileName.toString(); + lastFileName = new Path(lastFileName + "_" + suffixId); + } + + appender = StorageManager.getFileStorageManager(context.getConf()).getAppender(meta, appenderSchema, lastFileName); - if (plan instanceof InsertNode) { - InsertNode createTableNode = (InsertNode) plan; - appender = StorageManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender(meta, - createTableNode.getTableSchema(), context.getOutputPath()); + if (suffixId > 0) { + LOG.info(prevFile + " exceeds " + SessionVars.MAX_OUTPUT_FILE_SIZE.keyname() + " (" + maxPerFileSize + " MB), " + + "The remain output will be written into " + lastFileName.toString()); + } } else { - appender = StorageManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender(meta, - outSchema, lastFileName); + appender = StorageManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender( + context.getTaskId(), meta, appenderSchema, context.getQueryContext().getStagingDir()); } appender.enableStats(); appender.init(); - - if (suffixId > 0) { - LOG.info(prevFile + " exceeds " + SessionVars.MAX_OUTPUT_FILE_SIZE.keyname() + " (" + maxPerFileSize + " MB), " + - "The remain output will be written into " + lastFileName.toString()); - } } /* (non-Javadoc) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 86fd355d72..77e3517e2c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.ConfigKey; import org.apache.tajo.OverridableConf; +import org.apache.tajo.QueryVars; import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; @@ -33,31 +34,6 @@ * QueryContent is a overridable config, and it provides a set of various configs for a query instance. */ public class QueryContext extends OverridableConf { - public static enum QueryVars implements ConfigKey { - COMMAND_TYPE, - STAGING_DIR, - OUTPUT_TABLE_NAME, - OUTPUT_TABLE_PATH, - OUTPUT_PARTITIONS, - OUTPUT_OVERWRITE, - OUTPUT_AS_DIRECTORY, - OUTPUT_PER_FILE_SIZE, - ; - - QueryVars() { - } - - @Override - public String keyname() { - return name().toLowerCase(); - } - - @Override - public ConfigType type() { - return ConfigType.QUERY; - } - } - public QueryContext(TajoConf conf) { super(conf, ConfigKey.ConfigType.QUERY); } @@ -116,7 +92,9 @@ public boolean hasOutputPath() { } public void setOutputPath(Path path) { - put(QueryVars.OUTPUT_TABLE_PATH, path.toUri().toString()); + if (path != null) { + put(QueryVars.OUTPUT_TABLE_PATH, path.toUri().toString()); + } } public Path getOutputPath() { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 4b619363cd..fb434ed3bf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -28,10 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; -import org.apache.tajo.QueryId; -import org.apache.tajo.QueryIdFactory; -import org.apache.tajo.SessionVars; -import org.apache.tajo.TajoConstants; +import org.apache.tajo.*; import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.JsonHelper; @@ -40,10 +37,8 @@ import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; @@ -61,6 +56,7 @@ import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.verifier.LogicalPlanVerifier; import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; @@ -301,11 +297,14 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, responseBuilder.setResultCode(ClientProtos.ResultCode.OK); } } else { // it requires distributed execution. So, the query is forwarded to a query master. - if (rootNode.getChild().getType() == NodeType.CREATE_TABLE) { - StoreType storeType = ((CreateTableNode)rootNode.getChild()).getStorageType(); - if (!StorageManager.getStorageManager(context.getConf(), storeType).canCreateAsSelect(storeType)) { + StoreType storeType = PlannerUtil.getStoreType(plan); + if (storeType != null) { + StorageManager sm = StorageManager.getStorageManager(context.getConf(), storeType); + StorageProperty storageProperty = sm.getStorageProperty(); + if (!storageProperty.isSupportsInsertInto()) { throw new VerifyException("Inserting into non-file storage is not supported."); } + sm.verifyTableCreation(rootNode.getChild()); } context.getSystemMetrics().counter("Query", "numDMLQuery").inc(); hookManager.doHooks(queryContext, plan); @@ -355,8 +354,7 @@ private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode } TaskAttemptContext taskAttemptContext = - new TaskAttemptContext(queryContext, null, null, (CatalogProtos.FragmentProto[]) null, stagingDir); - taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000")); + new TaskAttemptContext(queryContext, null, null, (CatalogProtos.FragmentProto[]) null, stagingResultDir); EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); StoreTableExec exec = new StoreTableExec(taskAttemptContext, insertNode, evalExprExec); @@ -739,7 +737,7 @@ public TableDesc createTable(QueryContext queryContext, String tableName, StoreT desc.setPartitionMethod(partitionDesc); } - StorageManager.getStorageManager(queryContext.getConf(), storeType).createTable(desc); + StorageManager.getStorageManager(queryContext.getConf(), storeType).createTable(desc, ifNotExists); if (catalog.createTable(desc)) { LOG.info("Table " + desc.getName() + " is created (" + desc.getStats().getNumBytes() + ")"); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index e01b66127b..bfd3bce260 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -19,9 +19,11 @@ package org.apache.tajo.master.querymaster; import com.google.common.collect.Maps; +import org.apache.avro.ipc.trace.TracePlugin.StorageType; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -32,29 +34,26 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryId; import org.apache.tajo.SessionVars; -import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.plan.logical.CreateTableNode; -import org.apache.tajo.plan.logical.InsertNode; -import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.event.*; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.TUtil; import java.io.IOException; -import java.text.NumberFormat; import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; @@ -372,8 +371,12 @@ public QueryState transition(Query query, QueryEvent queryEvent) { QueryCompletedEvent subQueryEvent = (QueryCompletedEvent) queryEvent; QueryState finalState; if (subQueryEvent.getState() == SubQueryState.SUCCEEDED) { - finalizeQuery(query, subQueryEvent); - finalState = QueryState.QUERY_SUCCEEDED; + boolean success = finalizeQuery(query, subQueryEvent); + if (success) { + finalState = QueryState.QUERY_SUCCEEDED; + } else { + finalState = QueryState.QUERY_ERROR; + } } else if (subQueryEvent.getState() == SubQueryState.FAILED) { finalState = QueryState.QUERY_FAILED; } else if (subQueryEvent.getState() == SubQueryState.KILLED) { @@ -381,323 +384,42 @@ public QueryState transition(Query query, QueryEvent queryEvent) { } else { finalState = QueryState.QUERY_ERROR; } + if (finalState != QueryState.QUERY_SUCCEEDED) { + SubQuery lastStage = query.getSubQuery(subQueryEvent.getExecutionBlockId()); + StoreType storeType = lastStage.getTableMeta().getStoreType(); + LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); + try { + StorageManager.getStorageManager(query.systemConf, storeType).queryFailed(rootNode.getChild()); + } catch (IOException e) { + LOG.warn(query.getId() + ", failed processing cleanup storage when query failed:" + e.getMessage(), e); + } + } query.eventHandler.handle(new QueryMasterQueryCompletedEvent(query.getId())); query.setFinishTime(); return finalState; } - private void finalizeQuery(Query query, QueryCompletedEvent event) { - MasterPlan masterPlan = query.getPlan(); + private boolean finalizeQuery(Query query, QueryCompletedEvent event) { + SubQuery lastStage = query.getSubQuery(event.getExecutionBlockId()); + StoreType storeType = lastStage.getTableMeta().getStoreType(); + try { + LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); + CatalogService catalog = lastStage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); + TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); - ExecutionBlock terminal = query.getPlan().getTerminalBlock(); - DataChannel finalChannel = masterPlan.getChannel(event.getExecutionBlockId(), terminal.getId()); - Path finalOutputDir = commitOutputData(query); + Path finalOutputDir = StorageManager.getStorageManager(query.systemConf, storeType) + .commitOutputData(query.context.getQueryContext(), lastStage.getId(), lastStage.getSchema(), tableDesc); - QueryHookExecutor hookExecutor = new QueryHookExecutor(query.context.getQueryMasterContext()); - try { - hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(), - finalOutputDir); + QueryHookExecutor hookExecutor = new QueryHookExecutor(query.context.getQueryMasterContext()); + hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(), finalOutputDir); + return true; } catch (Exception e) { + LOG.error(e.getMessage(), e); query.eventHandler.handle(new QueryDiagnosticsUpdateEvent(query.id, ExceptionUtils.getStackTrace(e))); + return false; } } - /** - * It moves a result data stored in a staging output dir into a final output dir. - */ - public Path commitOutputData(Query query) { - QueryContext queryContext = query.context.getQueryContext(); - Path stagingResultDir = new Path(queryContext.getStagingDir(), TajoConstants.RESULT_DIR_NAME); - Path finalOutputDir; - if (queryContext.hasOutputPath()) { - finalOutputDir = queryContext.getOutputPath(); - try { - FileSystem fs = stagingResultDir.getFileSystem(query.systemConf); - - if (queryContext.isOutputOverwrite()) { // INSERT OVERWRITE INTO - - // It moves the original table into the temporary location. - // Then it moves the new result table into the original table location. - // Upon failed, it recovers the original table if possible. - boolean movedToOldTable = false; - boolean committed = false; - Path oldTableDir = new Path(queryContext.getStagingDir(), TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); - - if (queryContext.hasPartition()) { - // This is a map for existing non-leaf directory to rename. A key is current directory and a value is - // renaming directory. - Map renameDirs = TUtil.newHashMap(); - // This is a map for recovering existing partition directory. A key is current directory and a value is - // temporary directory to back up. - Map recoveryDirs = TUtil.newHashMap(); - - try { - if (!fs.exists(finalOutputDir)) { - fs.mkdirs(finalOutputDir); - } - - visitPartitionedDirectory(fs, stagingResultDir, finalOutputDir, stagingResultDir.toString(), - renameDirs, oldTableDir); - - // Rename target partition directories - for(Map.Entry entry : renameDirs.entrySet()) { - // Backup existing data files for recovering - if (fs.exists(entry.getValue())) { - String recoveryPathString = entry.getValue().toString().replaceAll(finalOutputDir.toString(), - oldTableDir.toString()); - Path recoveryPath = new Path(recoveryPathString); - fs.rename(entry.getValue(), recoveryPath); - fs.exists(recoveryPath); - recoveryDirs.put(entry.getValue(), recoveryPath); - } - // Delete existing directory - fs.delete(entry.getValue(), true); - // Rename staging directory to final output directory - fs.rename(entry.getKey(), entry.getValue()); - } - - } catch (IOException ioe) { - // Remove created dirs - for(Map.Entry entry : renameDirs.entrySet()) { - fs.delete(entry.getValue(), true); - } - - // Recovery renamed dirs - for(Map.Entry entry : recoveryDirs.entrySet()) { - fs.delete(entry.getValue(), true); - fs.rename(entry.getValue(), entry.getKey()); - } - throw new IOException(ioe.getMessage()); - } - } else { - try { - if (fs.exists(finalOutputDir)) { - fs.rename(finalOutputDir, oldTableDir); - movedToOldTable = fs.exists(oldTableDir); - } else { // if the parent does not exist, make its parent directory. - fs.mkdirs(finalOutputDir.getParent()); - } - - fs.rename(stagingResultDir, finalOutputDir); - committed = fs.exists(finalOutputDir); - } catch (IOException ioe) { - // recover the old table - if (movedToOldTable && !committed) { - fs.rename(oldTableDir, finalOutputDir); - } - } - } - } else { - NodeType queryType = queryContext.getCommandType(); - - if (queryType == NodeType.INSERT) { // INSERT INTO an existing table - - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(3); - - if (queryContext.hasPartition()) { - for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { - if (eachFile.isFile()) { - LOG.warn("Partition table can't have file in a staging dir: " + eachFile.getPath()); - continue; - } - moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, -1); - } - } else { - int maxSeq = StorageUtil.getMaxFileSequence(fs, finalOutputDir, false) + 1; - for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { - moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, maxSeq++); - } - } - // checking all file moved and remove empty dir - verifyAllFileMoved(fs, stagingResultDir); - FileStatus[] files = fs.listStatus(stagingResultDir); - if (files != null && files.length != 0) { - for (FileStatus eachFile: files) { - LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); - } - } - } else { // CREATE TABLE AS SELECT (CTAS) - fs.rename(stagingResultDir, finalOutputDir); - LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); - } - } - } catch (IOException e) { - // TODO report to client - e.printStackTrace(); - } - } else { - finalOutputDir = new Path(queryContext.getStagingDir(), TajoConstants.RESULT_DIR_NAME); - } - - return finalOutputDir; - } - - /** - * This method sets a rename map which includes renamed staging directory to final output directory recursively. - * If there exists some data files, this delete it for duplicate data. - * - * - * @param fs - * @param stagingPath - * @param outputPath - * @param stagingParentPathString - * @throws IOException - */ - private void visitPartitionedDirectory(FileSystem fs, Path stagingPath, Path outputPath, - String stagingParentPathString, - Map renameDirs, Path oldTableDir) throws IOException { - FileStatus[] files = fs.listStatus(stagingPath); - - for(FileStatus eachFile : files) { - if (eachFile.isDirectory()) { - Path oldPath = eachFile.getPath(); - - // Make recover directory. - String recoverPathString = oldPath.toString().replaceAll(stagingParentPathString, - oldTableDir.toString()); - Path recoveryPath = new Path(recoverPathString); - if (!fs.exists(recoveryPath)) { - fs.mkdirs(recoveryPath); - } - - visitPartitionedDirectory(fs, eachFile.getPath(), outputPath, stagingParentPathString, - renameDirs, oldTableDir); - // Find last order partition for renaming - String newPathString = oldPath.toString().replaceAll(stagingParentPathString, - outputPath.toString()); - Path newPath = new Path(newPathString); - if (!isLeafDirectory(fs, eachFile.getPath())) { - renameDirs.put(eachFile.getPath(), newPath); - } else { - if (!fs.exists(newPath)) { - fs.mkdirs(newPath); - } - } - } - } - } - - private boolean isLeafDirectory(FileSystem fs, Path path) throws IOException { - boolean retValue = false; - - FileStatus[] files = fs.listStatus(path); - for (FileStatus file : files) { - if (fs.isDirectory(file.getPath())) { - retValue = true; - break; - } - } - - return retValue; - } - - private boolean verifyAllFileMoved(FileSystem fs, Path stagingPath) throws IOException { - FileStatus[] files = fs.listStatus(stagingPath); - if (files != null && files.length != 0) { - for (FileStatus eachFile: files) { - if (eachFile.isFile()) { - LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); - return false; - } else { - if (verifyAllFileMoved(fs, eachFile.getPath())) { - fs.delete(eachFile.getPath(), false); - } else { - return false; - } - } - } - } - - return true; - } - - /** - * Attach the sequence number to a path. - * - * @param path Path - * @param seq sequence number - * @param nf Number format - * @return New path attached with sequence number - * @throws IOException - */ - private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IOException { - String[] tokens = path.getName().split("-"); - if (tokens.length != 4) { - throw new IOException("Wrong result file name:" + path); - } - return tokens[0] + "-" + tokens[1] + "-" + tokens[2] + "-" + nf.format(seq); - } - - /** - * Attach the sequence number to the output file name and than move the file into the final result path. - * - * @param fs FileSystem - * @param stagingResultDir The staging result dir - * @param fileStatus The file status - * @param finalOutputPath Final output path - * @param nf Number format - * @param fileSeq The sequence number - * @throws IOException - */ - private void moveResultFromStageToFinal(FileSystem fs, Path stagingResultDir, - FileStatus fileStatus, Path finalOutputPath, - NumberFormat nf, - int fileSeq) throws IOException { - if (fileStatus.isDirectory()) { - String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); - if (subPath != null) { - Path finalSubPath = new Path(finalOutputPath, subPath); - if (!fs.exists(finalSubPath)) { - fs.mkdirs(finalSubPath); - } - int maxSeq = StorageUtil.getMaxFileSequence(fs, finalSubPath, false); - for (FileStatus eachFile : fs.listStatus(fileStatus.getPath())) { - moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputPath, nf, ++maxSeq); - } - } else { - throw new IOException("Wrong staging dir:" + stagingResultDir + "," + fileStatus.getPath()); - } - } else { - String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); - if (subPath != null) { - Path finalSubPath = new Path(finalOutputPath, subPath); - finalSubPath = new Path(finalSubPath.getParent(), replaceFileNameSeq(finalSubPath, fileSeq, nf)); - if (!fs.exists(finalSubPath.getParent())) { - fs.mkdirs(finalSubPath.getParent()); - } - if (fs.exists(finalSubPath)) { - throw new IOException("Already exists data file:" + finalSubPath); - } - boolean success = fs.rename(fileStatus.getPath(), finalSubPath); - if (success) { - LOG.info("Moving staging file[" + fileStatus.getPath() + "] + " + - "to final output[" + finalSubPath + "]"); - } else { - LOG.error("Can't move staging file[" + fileStatus.getPath() + "] + " + - "to final output[" + finalSubPath + "]"); - } - } - } - } - - private String extractSubPath(Path parentPath, Path childPath) { - String parentPathStr = parentPath.toUri().getPath(); - String childPathStr = childPath.toUri().getPath(); - - if (parentPathStr.length() > childPathStr.length()) { - return null; - } - - int index = childPathStr.indexOf(parentPathStr); - if (index != 0) { - return null; - } - - return childPathStr.substring(parentPathStr.length() + 1); - } - private static interface QueryHook { boolean isEligible(QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir); void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index ef301353d9..6d6a9a2727 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -35,10 +35,14 @@ import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.logical.LogicalRootNode; +import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.rules.AddSortForInsertRewriter; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.plan.logical.LogicalNode; @@ -54,10 +58,12 @@ import org.apache.tajo.master.event.*; import org.apache.tajo.master.rm.TajoWorkerResourceManager; import org.apache.tajo.master.session.Session; +import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.rpc.CallFuture; import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.RpcConnectionPool; import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.StorageProperty; import org.apache.tajo.util.HAServiceUtil; import org.apache.tajo.util.metrics.TajoMetrics; import org.apache.tajo.util.metrics.reporter.MetricsConsoleReporter; @@ -345,6 +351,8 @@ private static boolean isTerminatedState(QueryState state) { } public synchronized void startQuery() { + StorageManager sm = null; + LogicalPlan plan = null; try { if (query != null) { LOG.warn("Query already started"); @@ -354,7 +362,24 @@ public synchronized void startQuery() { LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(systemConf); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); - LogicalPlan plan = planner.createPlan(queryContext, expr); + plan = planner.createPlan(queryContext, expr); + + StoreType storeType = PlannerUtil.getStoreType(plan); + if (storeType != null) { + sm = StorageManager.getStorageManager(systemConf, storeType); + StorageProperty storageProperty = sm.getStorageProperty(); + if (storageProperty.isSortedInsert()) { + String tableName = PlannerUtil.getStoreTableName(plan); + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); + if (tableDesc == null) { + throw new VerifyException("Can't get table meta data from catalog: " + tableName); + } + optimizer.addRuleAfterToJoinOpt(new AddSortForInsertRewriter( + getQueryTaskContext().getQueryContext(), tableDesc, sm.getIndexColumns(tableDesc))); + } + } + optimizer.optimize(queryContext, plan); GlobalEngine.DistributedQueryHookManager hookManager = new GlobalEngine.DistributedQueryHookManager(); @@ -389,6 +414,15 @@ public synchronized void startQuery() { } catch (Throwable t) { LOG.error(t.getMessage(), t); initError = t; + + if (plan != null && sm != null) { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + try { + sm.queryFailed(rootNode.getChild()); + } catch (IOException e) { + LOG.warn(query.getId() + ", failed processing cleanup storage when query failed:" + e.getMessage(), e); + } + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index c9ea32acac..f558433a0d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; +import org.apache.tajo.ConfigKey; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; @@ -45,9 +46,11 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty; import org.apache.tajo.master.TaskSchedulerContext; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; +import org.apache.tajo.plan.logical.SortNode.SortRangeType; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.RowStoreUtil; @@ -650,39 +653,58 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo SortSpec [] sortSpecs = sortNode.getSortKeys(); Schema sortSchema = new Schema(channel.getShuffleKeys()); + TupleRange[] ranges; + int determinedTaskNum; + // calculate the number of maximum query ranges TableStats totalStat = computeChildBlocksStats(subQuery.getContext(), masterPlan, subQuery.getId()); // If there is an empty table in inner join, it should return zero rows. - if (totalStat.getNumBytes() == 0 && totalStat.getColumnStats().size() == 0 ) { + if (totalStat.getNumBytes() == 0 && totalStat.getColumnStats().size() == 0) { return; } TupleRange mergedRange = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats(), false); - RangePartitionAlgorithm partitioner = new UniformRangePartition(mergedRange, sortSpecs); - BigInteger card = partitioner.getTotalCardinality(); - // if the number of the range cardinality is less than the desired number of tasks, - // we set the the number of tasks to the number of range cardinality. - int determinedTaskNum; - if (card.compareTo(BigInteger.valueOf(maxNum)) < 0) { - LOG.info(subQuery.getId() + ", The range cardinality (" + card - + ") is less then the desired number of tasks (" + maxNum + ")"); - determinedTaskNum = card.intValue(); + if (sortNode.getSortRangeType() == SortRangeType.USING_STORAGE_MANAGER) { + StoreType storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); + CatalogService catalog = subQuery.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); + LogicalRootNode rootNode = masterPlan.getLogicalPlan().getRootBlock().getRoot(); + TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); + if (tableDesc == null) { + throw new IOException("Can't get table meta data from catalog: " + + PlannerUtil.getStoreTableName(masterPlan.getLogicalPlan())); + } + ranges = StorageManager.getStorageManager(subQuery.getContext().getConf(), storeType) + .getInsertSortRanges(subQuery.getContext().getQueryContext(), tableDesc, + sortNode.getInSchema(), sortSpecs, + mergedRange); + determinedTaskNum = ranges.length; } else { - determinedTaskNum = maxNum; - } + RangePartitionAlgorithm partitioner = new UniformRangePartition(mergedRange, sortSpecs); + BigInteger card = partitioner.getTotalCardinality(); + + // if the number of the range cardinality is less than the desired number of tasks, + // we set the the number of tasks to the number of range cardinality. + if (card.compareTo(BigInteger.valueOf(maxNum)) < 0) { + LOG.info(subQuery.getId() + ", The range cardinality (" + card + + ") is less then the desired number of tasks (" + maxNum + ")"); + determinedTaskNum = card.intValue(); + } else { + determinedTaskNum = maxNum; + } - LOG.info(subQuery.getId() + ", Try to divide " + mergedRange + " into " + determinedTaskNum + - " sub ranges (total units: " + determinedTaskNum + ")"); - TupleRange [] ranges = partitioner.partition(determinedTaskNum); - if (ranges == null || ranges.length == 0) { - LOG.warn(subQuery.getId() + " no range infos."); - } - TupleUtil.setMaxRangeIfNull(sortSpecs, sortSchema, totalStat.getColumnStats(), ranges); - if (LOG.isDebugEnabled()) { - if (ranges != null) { - for (TupleRange eachRange : ranges) { - LOG.debug(subQuery.getId() + " range: " + eachRange.getStart() + " ~ " + eachRange.getEnd()); + LOG.info(subQuery.getId() + ", Try to divide " + mergedRange + " into " + determinedTaskNum + + " sub ranges (total units: " + determinedTaskNum + ")"); + ranges = partitioner.partition(determinedTaskNum); + if (ranges == null || ranges.length == 0) { + LOG.warn(subQuery.getId() + " no range infos."); + } + TupleUtil.setMaxRangeIfNull(sortSpecs, sortSchema, totalStat.getColumnStats(), ranges); + if (LOG.isDebugEnabled()) { + if (ranges != null) { + for (TupleRange eachRange : ranges) { + LOG.debug(subQuery.getId() + " range: " + eachRange.getStart() + " ~ " + eachRange.getEnd()); + } } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 2a489f8b0d..17c1332d29 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -38,6 +38,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; @@ -604,14 +605,14 @@ private void finalizeStats() { } DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0); - // get default or store type - CatalogProtos.StoreType storeType = CatalogProtos.StoreType.CSV; // default setting // if store plan (i.e., CREATE or INSERT OVERWRITE) - StoreTableNode storeTableNode = PlannerUtil.findTopNode(getBlock().getPlan(), NodeType.STORE); - if (storeTableNode != null) { - storeType = storeTableNode.getStorageType(); + StoreType storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); + if (storeType == null) { + // get default or store type + storeType = StoreType.CSV; } + schema = channel.getSchema(); meta = CatalogUtil.newTableMeta(storeType, new KeyValueSet()); inputStatistics = statsArray[0]; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 3858c96a69..2bf0d76d2f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -106,39 +106,6 @@ public class Task { private Schema finalSchema = null; private TupleComparator sortComp = null; - static final String OUTPUT_FILE_PREFIX="part-"; - static final ThreadLocal OUTPUT_FILE_FORMAT_SUBQUERY = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(2); - return fmt; - } - }; - static final ThreadLocal OUTPUT_FILE_FORMAT_TASK = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(6); - return fmt; - } - }; - - static final ThreadLocal OUTPUT_FILE_FORMAT_SEQ = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(3); - return fmt; - } - }; - public Task(String taskRunnerId, Path baseDir, QueryUnitAttemptId taskId, @@ -187,16 +154,6 @@ public Task(String taskRunnerId, this.finalSchema = PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()); this.sortComp = new TupleComparator(finalSchema, sortNode.getSortKeys()); } - } else { - // The final result of a task will be written in a file named part-ss-nnnnnnn, - // where ss is the subquery id associated with this task, and nnnnnn is the task id. - Path outFilePath = StorageUtil.concatPath(queryContext.getStagingDir(), TajoConstants.RESULT_DIR_NAME, - OUTPUT_FILE_PREFIX + - OUTPUT_FILE_FORMAT_SUBQUERY.get().format(taskId.getQueryUnitId().getExecutionBlockId().getId()) + "-" + - OUTPUT_FILE_FORMAT_TASK.get().format(taskId.getQueryUnitId().getId()) + "-" + - OUTPUT_FILE_FORMAT_SEQ.get().format(0)); - LOG.info("Output File Path: " + outFilePath); - context.setOutputPath(outFilePath); } this.localChunks = Collections.synchronizedList(new ArrayList()); @@ -528,10 +485,6 @@ public TaskHistory createTaskHistory() { taskHistory = new TaskHistory(getTaskId(), getStatus(), context.getProgress(), startTime, finishTime, reloadInputStats()); - if (context.getOutputPath() != null) { - taskHistory.setOutputPath(context.getOutputPath().toString()); - } - if (context.getWorkDir() != null) { taskHistory.setWorkingPath(context.getWorkDir().toString()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 99976d8e44..6e7a1cb98a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -72,7 +72,7 @@ public class TaskAttemptContext { private File fetchIn; private boolean stopped = false; private boolean interQuery = false; - private Path outputPath; +// private Path outputPath; private DataChannel dataChannel; private Enforcer enforcer; private QueryContext queryContext; @@ -198,13 +198,13 @@ public void setInterQuery() { this.interQuery = true; } - public void setOutputPath(Path outputPath) { - this.outputPath = outputPath; - } - - public Path getOutputPath() { - return this.outputPath; - } +// public void setOutputPath(Path outputPath) { +// this.outputPath = outputPath; +// } +// +// public Path getOutputPath() { +// return this.outputPath; +// } public boolean isInterQuery() { return this.interQuery; diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index a272b15ca5..9431b26cb8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -302,7 +302,7 @@ public ResultSet executeQuery() throws Exception { return executeFile(getMethodName() + ".sql"); } - private String getMethodName() { + protected String getMethodName() { String methodName = name.getMethodName(); // In the case of parameter execution name's pattern is methodName[0] if (methodName.endsWith("]")) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 08ccf0b61c..592b6fd3d9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -435,7 +435,6 @@ public final void testStorePlan() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); - ctx.setOutputPath(new Path(workDir, "grouped1")); Expr context = analyzer.parse(CreateTableAsStmts[0]); LogicalPlan plan = planner.createPlan(defaultContext, context); @@ -451,7 +450,7 @@ public final void testStorePlan() throws IOException, PlanningException { exec.close(); Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), - ctx.getOutputPath()); + new Path(workDir, "grouped1")); scanner.init(); Tuple tuple; int i = 0; @@ -487,7 +486,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); - ctx.setOutputPath(new Path(workDir, "maxOutput")); + Path outputPath = new Path(workDir, "maxOutput"); Expr context = analyzer.parse(CreateTableAsStmts[3]); @@ -503,13 +502,13 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann // checking the number of punctuated files int expectedFileNum = (int) (stats.getNumBytes() / (float) StorageUnit.MB); - FileSystem fs = ctx.getOutputPath().getFileSystem(conf); - FileStatus [] statuses = fs.listStatus(ctx.getOutputPath().getParent()); + FileSystem fs = outputPath.getFileSystem(conf); + FileStatus [] statuses = fs.listStatus(outputPath.getParent()); assertEquals(expectedFileNum, statuses.length); // checking the file contents long totalNum = 0; - for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { + for (FileStatus status : fs.listStatus(outputPath.getParent())) { Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner( CatalogUtil.newTableMeta(StoreType.CSV), rootNode.getOutSchema(), @@ -533,7 +532,6 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); - ctx.setOutputPath(new Path(workDir, "grouped2")); Expr context = analyzer.parse(CreateTableAsStmts[1]); LogicalPlan plan = planner.createPlan(defaultContext, context); @@ -548,7 +546,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio exec.close(); Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), - ctx.getOutputPath()); + new Path(workDir, "grouped2")); scanner.init(); Tuple tuple; int i = 0; @@ -574,7 +572,6 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); - ctx.setOutputPath(new Path(workDir, "grouped3")); Expr context = analyzer.parse(CreateTableAsStmts[2]); LogicalPlan plan = planner.createPlan(defaultContext, context); @@ -601,7 +598,6 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); - ctx.setOutputPath(new Path(workDir, "grouped4")); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -625,7 +621,6 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); - ctx.setOutputPath(new Path(workDir, "grouped5")); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -714,7 +709,6 @@ public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, // Preparing task context TaskAttemptContext ctx = new TaskAttemptContext(queryContext, id, new FileFragment[] { frags[0] }, workDir); - ctx.setOutputPath(new Path(workDir, "part-01-000000")); // SortBasedColumnPartitionStoreExec will be chosen by default. ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(CreateTableAsStmts[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 368b5d479f..bd1857364a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -24,16 +24,23 @@ import org.apache.hadoop.hbase.client.*; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.hbase.HBaseBinarySerializerDeserializer; import org.apache.tajo.storage.hbase.HBaseFragment; import org.apache.tajo.storage.hbase.HBaseStorageManager; +import org.apache.tajo.storage.hbase.HBaseTextSerializerDeserializer; import org.apache.tajo.util.Bytes; +import org.apache.tajo.util.KeyValueSet; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -42,7 +49,9 @@ import java.net.InetAddress; import java.sql.ResultSet; import java.text.DecimalFormat; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import static org.junit.Assert.*; @@ -527,15 +536,264 @@ public void testJoin() throws Exception { } } + @Test + public void testInsertInto() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int4) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + executeString("insert into hbase_mapped_table " + + "select l_orderkey::text, l_shipdate, l_returnflag, l_suppkey from default.lineitem "); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scan.addFamily(Bytes.toBytes("col2")); + scan.addFamily(Bytes.toBytes("col3")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col1"), Bytes.toBytes("col2"), Bytes.toBytes("col3")}, + new byte[][]{null, Bytes.toBytes("a"), null, Bytes.toBytes("b")}, + new boolean[]{false, false, false, true}, tableDesc.getSchema())); + } finally { + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + + @Test + public void testInsertIntoMultiRegion() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " + + "'hbase.split.rowkeys'='010,040,060,080', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", Type.TEXT); + schema.addColumn("name", Type.TEXT); + List datas = new ArrayList(); + DecimalFormat df = new DecimalFormat("000"); + for (int i = 99; i >= 0; i--) { + datas.add(df.format(i) + "|value" + i); + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("insert into hbase_mapped_table " + + "select id, name from base_table "); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col1")}, + new byte[][]{null, Bytes.toBytes("a")}, + new boolean[]{false, false}, tableDesc.getSchema())); + } finally { + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + + @Test + public void testInsertIntoRowField() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk1 text, rk2 text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a,col2:,col3:b', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + executeString("insert into hbase_mapped_table " + + "select l_orderkey::text, l_partkey::text, l_shipdate, l_returnflag, l_suppkey::text from default.lineitem "); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scan.addFamily(Bytes.toBytes("col2")); + scan.addFamily(Bytes.toBytes("col3")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col1"), Bytes.toBytes("col2"), Bytes.toBytes("col3")}, + new byte[][]{null, Bytes.toBytes("a"), Bytes.toBytes(""), Bytes.toBytes("b")}, + new boolean[]{false, false, false, false}, tableDesc.getSchema())); + } finally { + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + @Test public void testCATS() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", Type.TEXT); + schema.addColumn("name", Type.TEXT); + List datas = new ArrayList(); + DecimalFormat df = new DecimalFormat("000"); + for (int i = 99; i >= 0; i--) { + datas.add(df.format(i) + "|value" + i); + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " + + "'hbase.split.rowkeys'='010,040,060,080', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')" + + " as " + + "select id, name from base_table" + ).close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + HTable htable = null; + ResultScanner scanner = null; try { - ResultSet res = executeQuery(); - fail("CATS not supported"); - } catch (Exception e) { - if (e.getMessage().indexOf("not supported") < 0) { - fail(e.getMessage()); + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col1")}, + new byte[][]{null, Bytes.toBytes("a")}, + new boolean[]{false, false}, tableDesc.getSchema())); + } finally { + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); } } } + + private String resultSetToString(ResultScanner scanner, + byte[][] cfNames, byte[][] qualifiers, + boolean[] binaries, + Schema schema) throws Exception { + StringBuilder sb = new StringBuilder(); + Result result = null; + while ( (result = scanner.next()) != null ) { + if (binaries[0]) { + sb.append(Bytes.toLong(result.getRow())); + } else { + sb.append(new String(result.getRow())); + } + + for (int i = 0; i < cfNames.length; i++) { + if (cfNames[i] == null) { + //rowkey + continue; + } + if (qualifiers[i] == null) { + Map values = result.getFamilyMap(cfNames[i]); + if (values == null) { + sb.append(", null"); + } else { + sb.append(", {"); + for (Map.Entry valueEntry: values.entrySet()) { + byte[] keyBytes = valueEntry.getKey(); + byte[] valueBytes = valueEntry.getValue(); + + if (binaries[i]) { + sb.append("\"").append(keyBytes == null ? "" : Bytes.toLong(keyBytes)).append("\""); + sb.append(": \"").append(HBaseBinarySerializerDeserializer.deserialize(schema.getColumn(i), valueBytes)).append("\""); + } else { + sb.append("\"").append(keyBytes == null ? "" : new String(keyBytes)).append("\""); + sb.append(": \"").append(HBaseTextSerializerDeserializer.deserialize(schema.getColumn(i), valueBytes)).append("\""); + } + } + sb.append("}"); + } + } else { + byte[] value = result.getValue(cfNames[i], qualifiers[i]); + if (value == null) { + sb.append(", null"); + } else { + if (binaries[i]) { + sb.append(", ").append(HBaseBinarySerializerDeserializer.deserialize(schema.getColumn(i), value)); + } else { + sb.append(", ").append(HBaseTextSerializerDeserializer.deserialize(schema.getColumn(i), value)); + } + } + } + } + sb.append("\n"); + } + + return sb.toString(); + } } diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index 7b7399ca02..2e0baed931 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -120,7 +120,7 @@ public void testGet() throws Exception { Path tableDir = StorageUtil.concatPath(testDir, "testGet", "table.csv"); fs.mkdirs(tableDir.getParent()); - Appender appender = sm.getAppender(employeeMeta, schema, tableDir); + Appender appender = ((FileStorageManager)sm).getAppender(employeeMeta, schema, tableDir); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -245,7 +245,7 @@ public void testGetFromDescendingOrder() throws Exception { TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); Path tablePath = StorageUtil.concatPath(testDir, "testGetFromDescendingOrder", "table.csv"); fs.mkdirs(tablePath.getParent()); - Appender appender = sm.getAppender(meta, schema, tablePath); + Appender appender = ((FileStorageManager)sm).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = (TEST_TUPLE - 1); i >= 0 ; i--) { diff --git a/tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql b/tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql deleted file mode 100644 index 051d360823..0000000000 --- a/tajo-core/src/test/resources/queries/TestHBaseTable/testCATS.sql +++ /dev/null @@ -1,4 +0,0 @@ -create table hbase_mapped_table (rk int8, col1 int8) -using hbase with ('table'='hbase_table', 'columns'=':key,col1:a#b') as -select l_orderkey, l_partkey -from default.lineitem diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result b/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result new file mode 100644 index 0000000000..72013f26e9 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99 diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result new file mode 100644 index 0000000000..e0c97ef359 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result @@ -0,0 +1,3 @@ +1, 1996-03-13, {"": "N"}, 7706 +2, 1997-01-28, {"": "N"}, 1191 +3, 1994-02-02, {"": "R"}, 1798 diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result new file mode 100644 index 0000000000..72013f26e9 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99 diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result new file mode 100644 index 0000000000..152b91bd92 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result @@ -0,0 +1,4 @@ +1_1, 1996-03-13, N, 7706 +2_2, 1997-01-28, N, 1191 +3_2, 1994-02-02, R, 1798 +3_3, 1993-11-09, R, 6540 diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index bcc3bce39c..750e64e48c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -44,6 +44,7 @@ import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import java.util.Stack; @@ -86,6 +87,12 @@ public LogicalOptimizer(TajoConf systemConf) { } } + public void addRuleAfterToJoinOpt(RewriteRule rewriteRule) { + if (rewriteRule != null) { + rulesAfterToJoinOpt.addRewriteRule(rewriteRule); + } + } + public LogicalNode optimize(LogicalPlan plan) throws PlanningException { return optimize(null, plan); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java index a697f9f176..3c69a62950 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java @@ -25,10 +25,16 @@ import org.apache.tajo.util.TUtil; public final class SortNode extends UnaryNode implements Cloneable { + public static enum SortRangeType { + USING_SHUFFLE_DATA, + USING_STORAGE_MANAGER + } @Expose private SortSpec [] sortKeys; + @Expose private SortRangeType sortRangeType; public SortNode(int pid) { super(pid, NodeType.SORT); + sortRangeType = SortRangeType.USING_SHUFFLE_DATA; } public void setSortSpecs(SortSpec[] sortSpecs) { @@ -91,4 +97,12 @@ public String toString() { return sb.toString()+"\n" + getChild().toString(); } + + public SortRangeType getSortRangeType() { + return sortRangeType; + } + + public void setSortRangeType(SortRangeType sortRangeType) { + this.sortRangeType = sortRangeType; + } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java new file mode 100644 index 0000000000..fd831c7a0d --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java @@ -0,0 +1,62 @@ +package org.apache.tajo.plan.rewrite.rules; + +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.logical.SortNode.SortRangeType; +import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.util.PlannerUtil; + +public class AddSortForInsertRewriter implements RewriteRule { + private int[] sortColumnIndexes; + + public AddSortForInsertRewriter(OverridableConf queryContext, TableDesc tableDesc, Column[] sortColumns) { + this.sortColumnIndexes = new int[sortColumns.length]; + + Schema tableSchema = tableDesc.getSchema(); + for (int i = 0; i < sortColumns.length; i++) { + sortColumnIndexes[i] = tableSchema.getColumnId(sortColumns[i].getQualifiedName()); + } + } + + @Override + public String getName() { + return "AddSortForInsertRewriter"; + } + + @Override + public boolean isEligible(LogicalPlan plan) { + StoreType storeType = PlannerUtil.getStoreType(plan); + + return storeType != null; + } + + @Override + public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + UnaryNode topNode = rootNode.getChild(); + + SortNode sortNode = plan.createNode(SortNode.class); + sortNode.setSortRangeType(SortRangeType.USING_STORAGE_MANAGER); + sortNode.setInSchema(topNode.getInSchema()); + sortNode.setOutSchema(topNode.getInSchema()); + + SortSpec[] sortSpecs = new SortSpec[sortColumnIndexes.length]; + for (int i = 0; i < sortColumnIndexes.length; i++) { + sortSpecs[i] = new SortSpec(topNode.getInSchema().getColumn(sortColumnIndexes[i]), true, true); + } + sortNode.setSortSpecs(sortSpecs); + + sortNode.setChild(topNode.getChild()); + topNode.setChild(sortNode); + plan.getRootBlock().registerNode(sortNode); + + return plan; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index b135d9b577..ee13677871 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -23,9 +23,7 @@ import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -37,6 +35,7 @@ import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import org.apache.tajo.util.TUtil; +import java.io.IOException; import java.util.*; public class PlannerUtil { @@ -792,4 +791,71 @@ public static boolean isFileStorageType(StoreType storageType) { return true; } } + + public static StoreType getStoreType(LogicalPlan plan) { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + NodeType nodeType = rootNode.getChild().getType(); + if (nodeType == NodeType.CREATE_TABLE) { + return ((CreateTableNode)rootNode.getChild()).getStorageType(); + } else if (nodeType == NodeType.INSERT) { + return ((InsertNode)rootNode.getChild()).getStorageType(); + } else { + return null; + } + } + + public static String getStoreTableName(LogicalPlan plan) { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + NodeType nodeType = rootNode.getChild().getType(); + if (nodeType == NodeType.CREATE_TABLE) { + return ((CreateTableNode)rootNode.getChild()).getTableName(); + } else if (nodeType == NodeType.INSERT) { + return ((InsertNode)rootNode.getChild()).getTableName(); + } else { + return null; + } + } + + public static TableDesc getTableDesc(CatalogService catalog, LogicalNode node) throws IOException { + if (node.getType() == NodeType.CREATE_TABLE) { + return createTableDesc((CreateTableNode)node); + } + String tableName = null; + if (node.getType() == NodeType.CREATE_TABLE) { + tableName = ((CreateTableNode)node).getTableName(); + } else if (node.getType() == NodeType.INSERT) { + tableName = ((InsertNode)node).getTableName(); + } else { + return null; + } + + if (tableName != null) { + String[] tableTokens = tableName.split("\\."); + if (tableTokens.length >= 2) { + if (catalog.existsTable(tableTokens[0], tableTokens[1])) { + return catalog.getTableDesc(tableTokens[0], tableTokens[1]); + } + } + } + return null; + } + + private static TableDesc createTableDesc(CreateTableNode createTableNode) { + TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions()); + + TableDesc tableDescTobeCreated = + new TableDesc( + createTableNode.getTableName(), + createTableNode.getTableSchema(), + meta, + createTableNode.getPath()); + + tableDescTobeCreated.setExternal(createTableNode.isExternal()); + + if (createTableNode.hasPartition()) { + tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod()); + } + + return tableDescTobeCreated; + } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 1bd6bb6f57..5356c57fe4 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -279,10 +279,7 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws context.state.addVerification(String.format("relation \"%s\" does not exist", qualifiedName)); return null; } - if (!PlannerUtil.isFileStorageType(table.getMeta().getStoreType())) { - context.state.addVerification("Inserting into non-file storage is not supported."); - return null; - } + if (table.hasPartition()) { int columnSize = table.getSchema().getColumns().size(); columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size(); diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 6ae82e5b55..c0a835a2ba 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -296,6 +296,12 @@ + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + com.google.protobuf protobuf-java @@ -320,6 +326,11 @@ parquet-format ${parquet.format.version} + + org.apache.hbase + hbase-server + ${hbase.version} + org.apache.hbase hbase-client diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java index e0156581b8..c4687273f9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java @@ -28,6 +28,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.*; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -76,9 +77,10 @@ public static class CSVAppender extends FileAppender { private NonSyncByteArrayOutputStream os = new NonSyncByteArrayOutputStream(BUFFER_SIZE); private SerializerDeserializer serde; - public CSVAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path) throws IOException { - super(conf, schema, meta, path); - this.fs = path.getFileSystem(conf); + public CSVAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId, + final Schema schema, final TableMeta meta, final Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); + this.fs = workDir.getFileSystem(conf); this.meta = meta; this.schema = schema; this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_DELIMITER, diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java index 04278e9174..117d3da736 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java @@ -18,28 +18,48 @@ package org.apache.tajo.storage; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.conf.TajoConf; import java.io.IOException; public abstract class FileAppender implements Appender { + private static final Log LOG = LogFactory.getLog(FileAppender.class); + protected boolean inited = false; protected final Configuration conf; protected final TableMeta meta; protected final Schema schema; - protected final Path path; + protected final Path workDir; + protected final QueryUnitAttemptId taskAttemptId; protected boolean enabledStats; - - public FileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) { + protected Path path; + + public FileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema, + TableMeta meta, Path workDir) { this.conf = conf; this.meta = meta; this.schema = schema; - this.path = path; + this.workDir = workDir; + this.taskAttemptId = taskAttemptId; + + try { + if (taskAttemptId != null) { + this.path = StorageManager.getFileStorageManager((TajoConf) conf).getAppenderFilePath(taskAttemptId, workDir); + } else { + this.path = workDir; + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } } public void init() throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index cef7f48a9d..5e54bb7344 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import org.apache.commons.logging.Log; @@ -26,31 +27,70 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.tajo.TajoConstants; +import org.apache.tajo.*; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.hbase.IndexPredication; import org.apache.tajo.util.*; -import java.io.FileNotFoundException; import java.io.IOException; +import java.text.NumberFormat; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; public class FileStorageManager extends StorageManager { private final Log LOG = LogFactory.getLog(FileStorageManager.class); + static final String OUTPUT_FILE_PREFIX="part-"; + static final ThreadLocal OUTPUT_FILE_FORMAT_SUBQUERY = + new ThreadLocal() { + @Override + public NumberFormat initialValue() { + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(2); + return fmt; + } + }; + static final ThreadLocal OUTPUT_FILE_FORMAT_TASK = + new ThreadLocal() { + @Override + public NumberFormat initialValue() { + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(6); + return fmt; + } + }; + + static final ThreadLocal OUTPUT_FILE_FORMAT_SEQ = + new ThreadLocal() { + @Override + public NumberFormat initialValue() { + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(3); + return fmt; + } + }; + protected FileSystem fs; protected Path tableBaseDir; protected boolean blocksMetadataEnabled; private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); + public FileStorageManager(StoreType storeType) { + super(storeType); + } + @Override protected void storageInit() throws IOException { this.tableBaseDir = TajoConf.getWarehouseDir(conf); @@ -110,47 +150,10 @@ public Path getTablePath(String tableName) { return new Path(tableBaseDir, tableName); } - public Appender getAppender(TableMeta meta, Schema schema, Path path) + @VisibleForTesting + public Appender getAppender(TableMeta meta, Schema schema, Path filePath) throws IOException { - Appender appender; - - Class appenderClass; - - String handlerName = meta.getStoreType().name().toLowerCase(); - appenderClass = APPENDER_HANDLER_CACHE.get(handlerName); - if (appenderClass == null) { - appenderClass = conf.getClass( - String.format("tajo.storage.appender-handler.%s.class", - meta.getStoreType().name().toLowerCase()), null, - FileAppender.class); - APPENDER_HANDLER_CACHE.put(handlerName, appenderClass); - } - - if (appenderClass == null) { - throw new IOException("Unknown Storage Type: " + meta.getStoreType()); - } - - appender = newAppenderInstance(appenderClass, conf, meta, schema, path); - - return appender; - } - - public TableMeta getTableMeta(Path tablePath) throws IOException { - TableMeta meta; - - FileSystem fs = tablePath.getFileSystem(conf); - Path tableMetaPath = new Path(tablePath, ".meta"); - if (!fs.exists(tableMetaPath)) { - throw new FileNotFoundException(".meta file not found in " + tablePath.toString()); - } - - FSDataInputStream tableMetaIn = fs.open(tableMetaPath); - - CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) org.apache.tajo.util.FileUtil.loadProto(tableMetaIn, - CatalogProtos.TableProto.getDefaultInstance()); - meta = new TableMeta(tableProto); - - return meta; + return getAppender(null, meta, schema, filePath); } public FileFragment[] split(String tableName) throws IOException { @@ -274,6 +277,23 @@ public boolean accept(Path p) { } }; + public Path getAppenderFilePath(QueryUnitAttemptId taskAttemptId, Path workDir) { + if (taskAttemptId == null) { + // For testcase + return workDir; + } + // The final result of a task will be written in a file named part-ss-nnnnnnn, + // where ss is the subquery id associated with this task, and nnnnnn is the task id. + Path outFilePath = StorageUtil.concatPath(workDir, TajoConstants.RESULT_DIR_NAME, + OUTPUT_FILE_PREFIX + + OUTPUT_FILE_FORMAT_SUBQUERY.get().format(taskAttemptId.getQueryUnitId().getExecutionBlockId().getId()) + "-" + + OUTPUT_FILE_FORMAT_TASK.get().format(taskAttemptId.getQueryUnitId().getId()) + "-" + + OUTPUT_FILE_FORMAT_SEQ.get().format(0)); + LOG.info("Output File Path: " + outFilePath); + + return outFilePath; + } + /** * Proxy PathFilter that accepts a path only if all filters given in the * constructor do. Used by the listPaths() to apply the built-in @@ -640,7 +660,7 @@ public List getSplits(String tableName, TableDesc table, ScanNode scan } @Override - public void createTable(TableDesc tableDesc) throws IOException { + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { if (!tableDesc.isExternal()) { String [] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); String databaseName = splitted[0]; @@ -695,11 +715,6 @@ public void purgeTable(TableDesc tableDesc) throws IOException { } } - @Override - public Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { - return null; - } - @Override public List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { // Listing table data file which is not empty. @@ -786,15 +801,334 @@ private void getNonZeroLengthDataFiles(FileSystem fs, Path path, List renameDirs = TUtil.newHashMap(); + // This is a map for recovering existing partition directory. A key is current directory and a value is + // temporary directory to back up. + Map recoveryDirs = TUtil.newHashMap(); + + try { + if (!fs.exists(finalOutputDir)) { + fs.mkdirs(finalOutputDir); + } + + visitPartitionedDirectory(fs, stagingResultDir, finalOutputDir, stagingResultDir.toString(), + renameDirs, oldTableDir); + + // Rename target partition directories + for(Map.Entry entry : renameDirs.entrySet()) { + // Backup existing data files for recovering + if (fs.exists(entry.getValue())) { + String recoveryPathString = entry.getValue().toString().replaceAll(finalOutputDir.toString(), + oldTableDir.toString()); + Path recoveryPath = new Path(recoveryPathString); + fs.rename(entry.getValue(), recoveryPath); + fs.exists(recoveryPath); + recoveryDirs.put(entry.getValue(), recoveryPath); + } + // Delete existing directory + fs.delete(entry.getValue(), true); + // Rename staging directory to final output directory + fs.rename(entry.getKey(), entry.getValue()); + } + + } catch (IOException ioe) { + // Remove created dirs + for(Map.Entry entry : renameDirs.entrySet()) { + fs.delete(entry.getValue(), true); + } + + // Recovery renamed dirs + for(Map.Entry entry : recoveryDirs.entrySet()) { + fs.delete(entry.getValue(), true); + fs.rename(entry.getValue(), entry.getKey()); + } + throw new IOException(ioe.getMessage()); + } + } else { + try { + if (fs.exists(finalOutputDir)) { + fs.rename(finalOutputDir, oldTableDir); + movedToOldTable = fs.exists(oldTableDir); + } else { // if the parent does not exist, make its parent directory. + fs.mkdirs(finalOutputDir.getParent()); + } + + fs.rename(stagingResultDir, finalOutputDir); + committed = fs.exists(finalOutputDir); + } catch (IOException ioe) { + // recover the old table + if (movedToOldTable && !committed) { + fs.rename(oldTableDir, finalOutputDir); + } + } + } + } else { + String queryType = queryContext.get(QueryVars.COMMAND_TYPE); + + if (queryType != null && queryType.equals(NodeType.INSERT.name())) { // INSERT INTO an existing table + + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(3); + + if (queryContext.get(QueryVars.OUTPUT_PARTITIONS) != null) { + for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { + if (eachFile.isFile()) { + LOG.warn("Partition table can't have file in a staging dir: " + eachFile.getPath()); + continue; + } + moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, -1); + } + } else { + int maxSeq = StorageUtil.getMaxFileSequence(fs, finalOutputDir, false) + 1; + for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { + moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, maxSeq++); + } + } + // checking all file moved and remove empty dir + verifyAllFileMoved(fs, stagingResultDir); + FileStatus[] files = fs.listStatus(stagingResultDir); + if (files != null && files.length != 0) { + for (FileStatus eachFile: files) { + LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); + } + } + } else { // CREATE TABLE AS SELECT (CTAS) + fs.rename(stagingResultDir, finalOutputDir); + LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); + } + } + } catch (IOException e) { + // TODO report to client + e.printStackTrace(); + } + } else { + finalOutputDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); + } + + return finalOutputDir; + } + + /** + * This method sets a rename map which includes renamed staging directory to final output directory recursively. + * If there exists some data files, this delete it for duplicate data. + * + * + * @param fs + * @param stagingPath + * @param outputPath + * @param stagingParentPathString + * @throws IOException + */ + private void visitPartitionedDirectory(FileSystem fs, Path stagingPath, Path outputPath, + String stagingParentPathString, + Map renameDirs, Path oldTableDir) throws IOException { + FileStatus[] files = fs.listStatus(stagingPath); + + for(FileStatus eachFile : files) { + if (eachFile.isDirectory()) { + Path oldPath = eachFile.getPath(); + + // Make recover directory. + String recoverPathString = oldPath.toString().replaceAll(stagingParentPathString, + oldTableDir.toString()); + Path recoveryPath = new Path(recoverPathString); + if (!fs.exists(recoveryPath)) { + fs.mkdirs(recoveryPath); + } + + visitPartitionedDirectory(fs, eachFile.getPath(), outputPath, stagingParentPathString, + renameDirs, oldTableDir); + // Find last order partition for renaming + String newPathString = oldPath.toString().replaceAll(stagingParentPathString, + outputPath.toString()); + Path newPath = new Path(newPathString); + if (!isLeafDirectory(fs, eachFile.getPath())) { + renameDirs.put(eachFile.getPath(), newPath); + } else { + if (!fs.exists(newPath)) { + fs.mkdirs(newPath); + } + } + } + } + } + + private boolean isLeafDirectory(FileSystem fs, Path path) throws IOException { + boolean retValue = false; + + FileStatus[] files = fs.listStatus(path); + for (FileStatus file : files) { + if (fs.isDirectory(file.getPath())) { + retValue = true; + break; + } + } + + return retValue; + } + + private boolean verifyAllFileMoved(FileSystem fs, Path stagingPath) throws IOException { + FileStatus[] files = fs.listStatus(stagingPath); + if (files != null && files.length != 0) { + for (FileStatus eachFile: files) { + if (eachFile.isFile()) { + LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); + return false; + } else { + if (verifyAllFileMoved(fs, eachFile.getPath())) { + fs.delete(eachFile.getPath(), false); + } else { + return false; + } + } + } + } + + return true; + } + + /** + * Attach the sequence number to the output file name and than move the file into the final result path. + * + * @param fs FileSystem + * @param stagingResultDir The staging result dir + * @param fileStatus The file status + * @param finalOutputPath Final output path + * @param nf Number format + * @param fileSeq The sequence number + * @throws IOException + */ + private void moveResultFromStageToFinal(FileSystem fs, Path stagingResultDir, + FileStatus fileStatus, Path finalOutputPath, + NumberFormat nf, + int fileSeq) throws IOException { + if (fileStatus.isDirectory()) { + String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); + if (subPath != null) { + Path finalSubPath = new Path(finalOutputPath, subPath); + if (!fs.exists(finalSubPath)) { + fs.mkdirs(finalSubPath); + } + int maxSeq = StorageUtil.getMaxFileSequence(fs, finalSubPath, false); + for (FileStatus eachFile : fs.listStatus(fileStatus.getPath())) { + moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputPath, nf, ++maxSeq); + } + } else { + throw new IOException("Wrong staging dir:" + stagingResultDir + "," + fileStatus.getPath()); + } + } else { + String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); + if (subPath != null) { + Path finalSubPath = new Path(finalOutputPath, subPath); + finalSubPath = new Path(finalSubPath.getParent(), replaceFileNameSeq(finalSubPath, fileSeq, nf)); + if (!fs.exists(finalSubPath.getParent())) { + fs.mkdirs(finalSubPath.getParent()); + } + if (fs.exists(finalSubPath)) { + throw new IOException("Already exists data file:" + finalSubPath); + } + boolean success = fs.rename(fileStatus.getPath(), finalSubPath); + if (success) { + LOG.info("Moving staging file[" + fileStatus.getPath() + "] + " + + "to final output[" + finalSubPath + "]"); + } else { + LOG.error("Can't move staging file[" + fileStatus.getPath() + "] + " + + "to final output[" + finalSubPath + "]"); + } + } + } + } + + private String extractSubPath(Path parentPath, Path childPath) { + String parentPathStr = parentPath.toUri().getPath(); + String childPathStr = childPath.toUri().getPath(); + + if (parentPathStr.length() > childPathStr.length()) { + return null; + } + + int index = childPathStr.indexOf(parentPathStr); + if (index != 0) { + return null; + } + + return childPathStr.substring(parentPathStr.length() + 1); + } + + /** + * Attach the sequence number to a path. + * + * @param path Path + * @param seq sequence number + * @param nf Number format + * @return New path attached with sequence number + * @throws IOException + */ + private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IOException { + String[] tokens = path.getName().split("-"); + if (tokens.length != 4) { + throw new IOException("Wrong result file name:" + path); + } + return tokens[0] + "-" + tokens[1] + "-" + tokens[2] + "-" + nf.format(seq); + } + + @Override + public Column[] getIndexColumns(TableDesc tableDesc) throws IOException { + return null; + } + + @Override + public void verifyTableCreation(LogicalNode node) throws IOException { + } + + @Override + public void queryFailed(LogicalNode node) throws IOException { + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, + Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) + throws IOException { + return null; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java index f72a5a13c8..a6fcbbda2f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; @@ -460,8 +461,9 @@ public static class RawFileAppender extends FileAppender { private TableStatistics stats; - public RawFileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException { - super(conf, schema, meta, path); + public RawFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); } public void init() throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java index 640cae20c0..efb23eb71c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -313,9 +314,10 @@ public static class RowFileAppender extends FileAppender { // statistics private TableStatistics stats; - public RowFileAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path) + public RowFileAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId, + final Schema schema, final TableMeta meta, final Path workDir) throws IOException { - super(conf, schema, meta, path); + super(conf, taskAttemptId, schema, meta, workDir); } public void init() throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 4280c9cb90..d85011f2e9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -18,34 +18,30 @@ package org.apache.tajo.storage; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.storage.hbase.HBaseStorageManager; -import org.apache.tajo.util.Bytes; -import org.apache.tajo.util.FileUtil; -import java.io.FileNotFoundException; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.URI; @@ -59,6 +55,7 @@ public abstract class StorageManager { private final Log LOG = LogFactory.getLog(StorageManager.class); protected TajoConf conf; + protected StoreType storeType; private static final Map storageManagers = Maps.newHashMap(); @@ -71,8 +68,8 @@ public abstract class StorageManager { /** * Cache of appender handlers for each storage type. */ - protected static final Map> APPENDER_HANDLER_CACHE - = new ConcurrentHashMap>(); + protected static final Map> APPENDER_HANDLER_CACHE + = new ConcurrentHashMap>(); /** * Cache of constructors for each class. Pins the classes so they @@ -81,17 +78,112 @@ public abstract class StorageManager { private static final Map, Constructor> CONSTRUCTOR_CACHE = new ConcurrentHashMap, Constructor>(); - protected abstract void storageInit() throws IOException ; - public abstract void createTable(TableDesc tableDesc) throws IOException; + public StorageManager(StoreType storeType) { + this.storeType = storeType; + } + /** + * + * @throws IOException + */ + protected abstract void storageInit() throws IOException; + + /** + * + * @param tableDesc + * @param ifNotExists + * @throws IOException + */ + public abstract void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException; + + /** + * + * @param tableDesc + * @throws IOException + */ public abstract void purgeTable(TableDesc tableDesc) throws IOException; + + /** + * + * @param fragmentId + * @param tableDesc + * @param scanNode + * @return + * @throws IOException + */ public abstract List getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException; + + /** + * + * @param tableDesc + * @param currentPage + * @param numFragments + * @return + * @throws IOException + */ public abstract List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException; - public abstract Column[] getIndexableColumns(TableDesc tableDesc) throws IOException; - public abstract boolean canCreateAsSelect(StoreType storeType); + + /** + * @return + */ + public abstract StorageProperty getStorageProperty(); + + /** + * Release storage manager resource + */ public abstract void closeStorageManager(); + /** + * It moves a result data stored in a staging output dir into a final output dir. + * @param queryContext + * @param finalEbId + * @param schema + * @param tableDesc + * @return + * @throws IOException + */ + public abstract Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, Schema schema, + TableDesc tableDesc) throws IOException; + + /** + * + * @param queryContext + * @param tableDesc + * @param inputSchema + * @param sortSpecs + * @return + * @throws IOException + */ + public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, + Schema inputSchema, SortSpec[] sortSpecs, + TupleRange dataRange) throws IOException; + + /** + * + * @param tableDesc + * @throws IOException + */ + public abstract Column[] getIndexColumns(TableDesc tableDesc) throws IOException; + + /** + * + * @param node + * @throws IOException + */ + public abstract void verifyTableCreation(LogicalNode node) throws IOException; + + /** + * + * @param node + * @throws IOException + */ + public abstract void queryFailed(LogicalNode node) throws IOException; + + public StoreType getStoreType() { + return storeType; + } + public void init(TajoConf tajoConf) throws IOException { this.conf = tajoConf; storageInit(); @@ -124,6 +216,14 @@ public static FileStorageManager getFileStorageManager(TajoConf tajoConf, Path w return (FileStorageManager) getStorageManager(copiedConf, StoreType.CSV, key); } + public static StorageManager getStorageManager(TajoConf tajoConf, String storeType) throws IOException { + if ("HBASE".equals(storeType)) { + return getStorageManager(tajoConf, StoreType.HBASE); + } else { + return getStorageManager(tajoConf, StoreType.CSV); + } + } + public static StorageManager getStorageManager(TajoConf tajoConf, StoreType storeType) throws IOException { return getStorageManager(tajoConf, storeType, null); } @@ -136,10 +236,10 @@ public static synchronized StorageManager getStorageManager ( if (manager == null) { switch (storeType) { case HBASE: - manager = new HBaseStorageManager(); + manager = new HBaseStorageManager(storeType); break; default: - manager = new FileStorageManager(); + manager = new FileStorageManager(storeType); } manager.init(conf); @@ -158,19 +258,18 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) thro return getScanner(meta, schema, fragment, schema); } - public Appender getAppender(TableMeta meta, Schema schema, Path path) + public Appender getAppender(QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) throws IOException { Appender appender; - Class appenderClass; + Class appenderClass; String handlerName = meta.getStoreType().name().toLowerCase(); appenderClass = APPENDER_HANDLER_CACHE.get(handlerName); if (appenderClass == null) { appenderClass = conf.getClass( String.format("tajo.storage.appender-handler.%s.class", - meta.getStoreType().name().toLowerCase()), null, - FileAppender.class); + meta.getStoreType().name().toLowerCase()), null, Appender.class); APPENDER_HANDLER_CACHE.put(handlerName, appenderClass); } @@ -178,7 +277,7 @@ public Appender getAppender(TableMeta meta, Schema schema, Path path) throw new IOException("Unknown Storage Type: " + meta.getStoreType()); } - appender = newAppenderInstance(appenderClass, conf, meta, schema, path); + appender = newAppenderInstance(appenderClass, conf, taskAttemptId, meta, schema, workDir); return appender; } @@ -192,6 +291,7 @@ public Appender getAppender(TableMeta meta, Schema schema, Path path) private static final Class[] DEFAULT_APPENDER_PARAMS = { Configuration.class, + QueryUnitAttemptId.class, Schema.class, TableMeta.class, Path.class @@ -221,8 +321,8 @@ public static T newScannerInstance(Class theClass, Configuration conf, Sc /** * create a scanner instance. */ - public static T newAppenderInstance(Class theClass, Configuration conf, TableMeta meta, Schema schema, - Path path) { + public static T newAppenderInstance(Class theClass, Configuration conf, QueryUnitAttemptId taskAttemptId, + TableMeta meta, Schema schema, Path workDir) { T result; try { Constructor meth = (Constructor) CONSTRUCTOR_CACHE.get(theClass); @@ -231,7 +331,7 @@ public static T newAppenderInstance(Class theClass, Configuration conf, T meth.setAccessible(true); CONSTRUCTOR_CACHE.put(theClass, meth); } - result = meth.newInstance(new Object[]{conf, schema, meta, path}); + result = meth.newInstance(new Object[]{conf, taskAttemptId, schema, meta, workDir}); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageProperty.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageProperty.java new file mode 100644 index 0000000000..6816d082c6 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageProperty.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage; + +public class StorageProperty { + private boolean supportsInsertInto; + private boolean sortedInsert; + + public boolean isSupportsInsertInto() { + return supportsInsertInto; + } + + public void setSupportsInsertInto(boolean supportsInsertInto) { + this.supportsInsertInto = supportsInsertInto; + } + + public boolean isSortedInsert() { + return sortedInsert; + } + + public void setSortedInsert(boolean sortedInsert) { + this.sortedInsert = sortedInsert; + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java index 6af8da0e98..9e1e7ea8aa 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.catalog.TableMeta; @@ -58,12 +59,13 @@ public class AvroAppender extends FileAppender { * @param conf Configuration properties. * @param schema The table schema. * @param meta The table metadata. - * @param path The path of the Parquet file to write to. + * @param workDir The path of the Parquet file to write to. */ public AvroAppender(Configuration conf, + QueryUnitAttemptId taskAttemptId, org.apache.tajo.catalog.Schema schema, - TableMeta meta, Path path) throws IOException { - super(conf, schema, meta, path); + TableMeta meta, Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); } /** diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java index 0c9b20edde..1fe81be5f5 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -32,6 +32,8 @@ public class ColumnMapping { private Schema schema; private char rowKeyDelimiter; + private String hbaseTableName; + private int[] rowKeyFieldIndexes; private boolean[] isRowKeyMappings; private boolean[] isBinaryColumns; @@ -47,6 +49,7 @@ public ColumnMapping(Schema schema, TableMeta tableMeta) throws IOException { } public void init() throws IOException { + hbaseTableName = tableMeta.getOption(HBaseStorageManager.META_TABLE_KEY); String delim = tableMeta.getOption(HBaseStorageManager.META_ROWKEY_DELIMITER, "").trim(); if (delim.length() > 0) { rowKeyDelimiter = delim.charAt(0); @@ -168,4 +171,8 @@ public static List> parseColumnMapping(TableMeta tableMeta) return columnMappings; } + + public String getHbaseTableName() { + return hbaseTableName; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java new file mode 100644 index 0000000000..1fb2c61b69 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.QueryUnitAttemptId; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.TableStatistics; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.Bytes; +import org.apache.tajo.util.TUtil; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class HBaseAppender implements Appender { + private static final Log LOG = LogFactory.getLog(HBaseAppender.class); + + private RecordWriter writer; + private Configuration conf; + private Schema schema; + private TableMeta meta; + private QueryUnitAttemptId taskAttemptId; + private Path stagingDir; + private boolean inited = false; + private TaskAttemptContext writerContext; + private int columnNum; + private ColumnMapping columnMapping; + private TableStatistics stats; + private boolean enabledStats; + + private byte[][][] mappingColumnFamilies; + private boolean[] isBinaryColumns; + private boolean[] isRowKeyMappings; + private int[] rowKeyFieldIndexes; + private int[] rowkeyColumnIndexes; + private char rowKeyDelimiter; + + private Path workingFilePath; + private FileOutputCommitter committer; + + public HBaseAppender (Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path stagingDir) { + this.conf = conf; + this.schema = schema; + this.meta = meta; + this.stagingDir = stagingDir; + this.taskAttemptId = taskAttemptId; + } + + @Override + public void init() throws IOException { + if (inited) { + throw new IllegalStateException("FileAppender is already initialized."); + } + inited = true; + if (enabledStats) { + stats = new TableStatistics(this.schema); + } + + columnMapping = new ColumnMapping(schema, meta); + mappingColumnFamilies = columnMapping.getMappingColumns(); + + isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + List rowkeyColumnIndexList = new ArrayList(); + for (int i = 0; i < isRowKeyMappings.length; i++) { + if (isRowKeyMappings[i]) { + rowkeyColumnIndexList.add(i); + } + } + rowkeyColumnIndexes = TUtil.toArray(rowkeyColumnIndexList); + + isBinaryColumns = columnMapping.getIsBinaryColumns(); + rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); + rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); + + this.columnNum = schema.size(); + + Configuration taskConf = new Configuration(); + taskConf.set(FileOutputFormat.OUTDIR, stagingDir.toString()); + + ExecutionBlockId ebId = taskAttemptId.getQueryUnitId().getExecutionBlockId(); + writerContext = new TaskAttemptContextImpl(taskConf, + new TaskAttemptID(ebId.getQueryId().toString(), ebId.getId(), TaskType.MAP, + taskAttemptId.getQueryUnitId().getId(), taskAttemptId.getId())); + + HFileOutputFormat2 hFileOutputFormat2 = new HFileOutputFormat2(); + try { + writer = hFileOutputFormat2.getRecordWriter(writerContext); + + committer = new FileOutputCommitter(FileOutputFormat.getOutputPath(writerContext), writerContext); + workingFilePath = committer.getWorkPath(); + } catch (InterruptedException e) { + throw new IOException(e.getMessage(), e); + } + + LOG.info("Created hbase file writer: " + workingFilePath); + } + + long totalNumBytes = 0; + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + ImmutableBytesWritable keyWritable = new ImmutableBytesWritable(); + @Override + public void addTuple(Tuple tuple) throws IOException { + Datum datum; + + byte[] rowkey; + // make rowkey + if (rowkeyColumnIndexes.length > 1) { + bout.reset(); + for (int i = 0; i < rowkeyColumnIndexes.length; i++) { + datum = tuple.get(rowkeyColumnIndexes[i]); + if (isBinaryColumns[rowkeyColumnIndexes[i]]) { + rowkey = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(rowkeyColumnIndexes[i]), datum); + } else { + rowkey = HBaseTextSerializerDeserializer.serialize(schema.getColumn(rowkeyColumnIndexes[i]), datum); + } + bout.write(rowkey); + if (i < rowkeyColumnIndexes.length - 1) { + bout.write(rowKeyDelimiter); + } + } + rowkey = bout.toByteArray(); + } else { + int index = rowkeyColumnIndexes[0]; + datum = tuple.get(index); + if (isBinaryColumns[index]) { + rowkey = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(index), datum); + } else { + rowkey = HBaseTextSerializerDeserializer.serialize(schema.getColumn(index), datum); + } + } + keyWritable.set(rowkey); + for (int i = 0; i < columnNum; i++) { + if (isRowKeyMappings[i]) { + continue; + } + datum = tuple.get(i); + byte[] value; + if (isBinaryColumns[i]) { + value = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(i), datum); + } else { + value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(i), datum); + } + KeyValue keyValue = new KeyValue(rowkey, mappingColumnFamilies[i][0], mappingColumnFamilies[i][1], value); + + try { + writer.write(keyWritable, keyValue); + } catch (InterruptedException e) { + LOG.error(e.getMessage(), e); + } + totalNumBytes += keyWritable.getLength() + keyValue.getLength(); + } + // Statistical section + if (enabledStats) { + stats.incrementRow(); + } + } + + @Override + public void flush() throws IOException { + } + + @Override + public long getEstimatedOutputSize() throws IOException { + // StoreTableExec uses this value as rolling file length + // Not rolling + return 0; + } + + @Override + public void close() throws IOException { + if (enabledStats) { + stats.setNumBytes(totalNumBytes); + } + if (writer != null) { + try { + writer.close(writerContext); + committer.commitTask(writerContext); + } catch (InterruptedException e) { + } + } + } + + @Override + public void enableStats() { + enabledStats = true; + } + + @Override + public TableStats getStats() { + if (enabledStats) { + return stats.getTableStat(); + } else { + return null; + } + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java index 9984a7caf7..39075b3522 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java @@ -28,35 +28,30 @@ import java.io.IOException; import java.io.OutputStream; -public class HBaseBinarySerializerDeserializer implements SerializerDeserializer { - @Override - public int serialize(Column col, Datum datum, OutputStream out, byte[] nullCharacters) throws IOException { - return 0; - } +public class HBaseBinarySerializerDeserializer { - @Override - public Datum deserialize(Column col, byte[] bytes, int offset, int length, byte[] nullCharacters) throws IOException { + public static Datum deserialize(Column col, byte[] bytes) throws IOException { Datum datum; switch (col.getDataType().getType()) { case INT1: case INT2: - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt2(Bytes.toShort(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt2(Bytes.toShort(bytes)); break; case INT4: - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes)); break; case INT8: - if (length == 4) { - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes, offset, length)); + if (bytes.length == 4) { + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes)); } else { - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes)); } break; case FLOAT4: - datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat4(Bytes.toFloat(bytes, offset)); + datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat4(Bytes.toFloat(bytes)); break; case FLOAT8: - datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat8(Bytes.toDouble(bytes, offset)); + datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat8(Bytes.toDouble(bytes)); break; case TEXT: datum = bytes == null ? NullDatum.get() : DatumFactory.createText(bytes); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java index 8e423fdcb1..6e39fe5834 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java @@ -43,15 +43,11 @@ public class HBaseLazyTuple implements Tuple, Cloneable { private boolean[] isBinaryColumns; private int[] rowKeyFieldIndexes; private char rowKeyDelimiter; - private HBaseTextSerializerDeserializer textSerde; - private HBaseBinarySerializerDeserializer binarySerde; private Column[] schemaColumns; public HBaseLazyTuple(ColumnMapping columnMapping, Column[] schemaColumns, int[] targetIndexes, - HBaseTextSerializerDeserializer textSerde, - HBaseBinarySerializerDeserializer binarySerde, Result result) { values = new Datum[schemaColumns.length]; mappingColumnFamilies = columnMapping.getMappingColumns(); @@ -62,8 +58,6 @@ public HBaseLazyTuple(ColumnMapping columnMapping, this.result = result; this.schemaColumns = schemaColumns; - this.textSerde = textSerde; - this.binarySerde = binarySerde; } @Override @@ -149,11 +143,9 @@ public Datum get(int fieldId) { if (entryValue != null) { try { if (isBinaryColumns[fieldId]) { - valueText = binarySerde.deserialize( - schemaColumns[fieldId], entryValue, 0, entryValue.length, null).asChars(); + valueText = HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue).asChars(); } else { - valueText = textSerde.deserialize( - schemaColumns[fieldId], entryValue, 0, entryValue.length, null).asChars(); + valueText = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue).asChars(); } } catch (Exception e) { LOG.error(e.getMessage(), e); @@ -182,9 +174,9 @@ public Datum get(int fieldId) { } else { try { if (isBinaryColumns[fieldId]) { - values[fieldId] = binarySerde.deserialize(schemaColumns[fieldId], value, 0, value.length, null); + values[fieldId] = HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], value); } else { - values[fieldId] = textSerde.deserialize(schemaColumns[fieldId], value, 0, value.length, null); + values[fieldId] = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], value); } } catch (Exception e) { LOG.error(e.getMessage(), e); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index 1bfa875878..b042520765 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -65,9 +65,6 @@ public class HBaseScanner implements Scanner { private ColumnMapping columnMapping; private int[] targetIndexes; - private HBaseTextSerializerDeserializer textSerde; - private HBaseBinarySerializerDeserializer binarySerde; - private int numRows = 0; public HBaseScanner (Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { @@ -93,9 +90,6 @@ public void init() throws IOException { } } - textSerde = new HBaseTextSerializerDeserializer(); - binarySerde = new HBaseBinarySerializerDeserializer(); - scanFetchSize = Integer.parseInt(meta.getOption("hbase.scanner.fetch,size", "" + DEFAULT_FETCH_SZIE)); if (targets == null) { targets = schema.toArray(); @@ -165,7 +159,7 @@ public Tuple next() throws IOException { Result result = scanResults[scanResultIndex++]; numRows++; - return new HBaseLazyTuple(columnMapping, schemaColumns, targetIndexes, textSerde, binarySerde, result); + return new HBaseLazyTuple(columnMapping, schemaColumns, targetIndexes, result); } @Override diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 04b941d4ea..73c990fbb9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -26,23 +26,27 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +import org.apache.hadoop.mapreduce.task.JobContextImpl; +import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.QueryVars; import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.expr.*; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.util.Bytes; -import org.apache.tajo.util.Pair; -import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.*; import java.io.BufferedReader; import java.io.IOException; @@ -62,6 +66,10 @@ public class HBaseStorageManager extends StorageManager { private Map connMap = new HashMap(); + public HBaseStorageManager (StoreType storeType) { + super(storeType); + } + @Override public void storageInit() throws IOException { } @@ -80,9 +88,15 @@ public void closeStorageManager() { } @Override - public void createTable(TableDesc tableDesc) throws IOException { - TableMeta tableMeta = tableDesc.getMeta(); + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { + createTable(tableDesc.getMeta(), tableDesc.getSchema(), tableDesc.isExternal(), ifNotExists); + TableStats stats = new TableStats(); + stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); + tableDesc.setStats(stats); + } + private void createTable(TableMeta tableMeta, Schema schema, + boolean isExternal, boolean ifNotExists) throws IOException { String hbaseTableName = tableMeta.getOption(META_TABLE_KEY, ""); if (hbaseTableName == null || hbaseTableName.trim().isEmpty()) { throw new IOException("HBase mapped table is required a '" + META_TABLE_KEY + "' attribute."); @@ -90,7 +104,7 @@ public void createTable(TableDesc tableDesc) throws IOException { TableName hTableName = TableName.valueOf(hbaseTableName); String columnMapping = tableMeta.getOption(META_COLUMNS_KEY, ""); - if (columnMapping != null && columnMapping.split(",").length > tableDesc.getSchema().size()) { + if (columnMapping != null && columnMapping.split(",").length > schema.size()) { throw new IOException("Columns property has more entry than Tajo table columns"); } @@ -98,7 +112,7 @@ public void createTable(TableDesc tableDesc) throws IOException { HBaseAdmin hAdmin = new HBaseAdmin(hConf); try { - if (tableDesc.isExternal()) { + if (isExternal) { // If tajo table is external table, only check validation. if (columnMapping == null || columnMapping.isEmpty()) { throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); @@ -125,10 +139,14 @@ public void createTable(TableDesc tableDesc) throws IOException { } } else { if (hAdmin.tableExists(hbaseTableName)) { - throw new IOException("HBase table [" + hbaseTableName + "] already exists."); + if (ifNotExists) { + return; + } else { + throw new IOException("HBase table [" + hbaseTableName + "] already exists."); + } } // Creating hbase table - HTableDescriptor hTableDescriptor = parseHTableDescriptor(tableDesc); + HTableDescriptor hTableDescriptor = parseHTableDescriptor(tableMeta, schema); byte[][] splitKeys = getSplitKeys(conf, tableMeta); if (splitKeys == null) { @@ -137,10 +155,6 @@ public void createTable(TableDesc tableDesc) throws IOException { hAdmin.createTable(hTableDescriptor, splitKeys); } } - - TableStats stats = new TableStats(); - stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); - tableDesc.setStats(stats); } finally { hAdmin.close(); } @@ -227,13 +241,13 @@ private static List getColumnFamilies(String columnMapping) { return columnFamilies; } - public static Configuration getHBaseConfiguration(TajoConf tajoConf, TableMeta tableMeta) throws IOException { + public static Configuration getHBaseConfiguration(Configuration conf, TableMeta tableMeta) throws IOException { String zkQuorum = tableMeta.getOption(META_ZK_QUORUM_KEY, ""); if (zkQuorum == null || zkQuorum.trim().isEmpty()) { throw new IOException("HBase mapped table is required a '" + META_ZK_QUORUM_KEY + "' attribute."); } - Configuration hbaseConf = (tajoConf == null) ? HBaseConfiguration.create() : HBaseConfiguration.create(tajoConf); + Configuration hbaseConf = (conf == null) ? HBaseConfiguration.create() : HBaseConfiguration.create(conf); hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum); for (Map.Entry eachOption: tableMeta.getOptions().getAllKeyValus().entrySet()) { @@ -245,9 +259,7 @@ public static Configuration getHBaseConfiguration(TajoConf tajoConf, TableMeta t return hbaseConf; } - public static HTableDescriptor parseHTableDescriptor(TableDesc tableDesc) throws IOException { - TableMeta tableMeta = tableDesc.getMeta(); - + public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema schema) throws IOException { String hbaseTableName = tableMeta.getOption(META_TABLE_KEY, ""); if (hbaseTableName == null || hbaseTableName.trim().isEmpty()) { throw new IOException("HBase mapped table is required a '" + META_TABLE_KEY + "' attribute."); @@ -255,7 +267,7 @@ public static HTableDescriptor parseHTableDescriptor(TableDesc tableDesc) throws TableName hTableName = TableName.valueOf(hbaseTableName); String columnMapping = tableMeta.getOption(META_COLUMNS_KEY, ""); - if (columnMapping != null && columnMapping.split(",").length > tableDesc.getSchema().size()) { + if (columnMapping != null && columnMapping.split(",").length > schema.size()) { throw new IOException("Columns property has more entry than Tajo table columns"); } HTableDescriptor hTableDescriptor = new HTableDescriptor(hTableName); @@ -263,7 +275,7 @@ public static HTableDescriptor parseHTableDescriptor(TableDesc tableDesc) throws Collection columnFamilies = getColumnFamilies(columnMapping); //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. if (columnFamilies.isEmpty()) { - for (Column eachColumn: tableDesc.getSchema().getColumns()) { + for (Column eachColumn: schema.getColumns()) { columnFamilies.add(eachColumn.getSimpleName()); } } @@ -280,7 +292,7 @@ public void purgeTable(TableDesc tableDesc) throws IOException { HBaseAdmin hAdmin = new HBaseAdmin(getHBaseConfiguration(conf, tableDesc.getMeta())); try { - HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc); + HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc.getMeta(), tableDesc.getSchema()); hAdmin.disableTable(hTableDesc.getName()); hAdmin.deleteTable(hTableDesc.getName()); } finally { @@ -288,8 +300,7 @@ public void purgeTable(TableDesc tableDesc) throws IOException { } } - @Override - public Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { + private Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); @@ -553,11 +564,6 @@ public List getNonForwardSplit(TableDesc tableDesc, int currentPage, i } } - @Override - public boolean canCreateAsSelect(StoreType storeType) { - return false; - } - public HConnection getConnection(Configuration hbaseConf) throws IOException { synchronized(connMap) { HConnectionKey key = new HConnectionKey(hbaseConf); @@ -845,4 +851,169 @@ public Pair getIndexablePredicateValue(Set evalNodes) { return null; } } + + @Override + public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, Schema schema, + TableDesc tableDesc) throws IOException { + if (tableDesc == null) { + throw new IOException("TableDesc is null while calling loadIncrementalHFiles: " + finalEbId); + } + Path finalOutputDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); + + Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); + hbaseConf.set("hbase.loadincremental.threads.max", "2"); + + JobContextImpl jobContext = new JobContextImpl(hbaseConf, + new JobID(finalEbId.getQueryId().toString(), finalEbId.getId())); + + FileOutputCommitter committer = new FileOutputCommitter(finalOutputDir, jobContext); + Path jobAttemptPath = committer.getJobAttemptPath(jobContext); + FileSystem fs = jobAttemptPath.getFileSystem(queryContext.getConf()); + if (!fs.exists(jobAttemptPath) || fs.listStatus(jobAttemptPath) == null) { + LOG.warn("No query attempt file in " + jobAttemptPath); + return finalOutputDir; + } + committer.commitJob(jobContext); + + String tableName = tableDesc.getMeta().getOption(HBaseStorageManager.META_TABLE_KEY); + + HTable htable = new HTable(hbaseConf, tableName); + try { + LoadIncrementalHFiles loadIncrementalHFiles = null; + try { + loadIncrementalHFiles = new LoadIncrementalHFiles(hbaseConf); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new IOException(e.getMessage(), e); + } + loadIncrementalHFiles.doBulkLoad(finalOutputDir, htable); + + return finalOutputDir; + } finally { + htable.close(); + } + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, + Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) + throws IOException { + int[] sortKeyIndexes = new int[sortSpecs.length]; + for (int i = 0; i < sortSpecs.length; i++) { + sortKeyIndexes[i] = inputSchema.getColumnId(sortSpecs[i].getSortKey().getQualifiedName()); + } + + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); + Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); + + HTable htable = new HTable(hbaseConf, columnMapping.getHbaseTableName()); + try { + byte[][] endKeys = htable.getEndKeys(); + if (endKeys.length == 1) { + return new TupleRange[]{dataRange}; + } + List tupleRanges = new ArrayList(endKeys.length); + + TupleComparator comparator = new TupleComparator(inputSchema, sortSpecs); + Tuple previousTuple = new VTuple(sortSpecs.length); + for (int i = 0; i < sortSpecs.length; i++) { + previousTuple.put(i, NullDatum.get()); + } + + for (byte[] eachEndKey: endKeys) { + Tuple endTuple = new VTuple(sortSpecs.length); + byte[][] rowKeyFields; + if (sortSpecs.length > 1) { + rowKeyFields = BytesUtils.splitPreserveAllTokens(eachEndKey, columnMapping.getRowKeyDelimiter()); + } else { + rowKeyFields = new byte[1][]; + rowKeyFields[0] = eachEndKey; + } + + for (int i = 0; i < sortSpecs.length; i++) { + if (columnMapping.getIsBinaryColumns()[sortKeyIndexes[i]]) { + endTuple.put(i, + HBaseBinarySerializerDeserializer.deserialize(inputSchema.getColumn(sortKeyIndexes[i]), + rowKeyFields[i])); + } else { + endTuple.put(i, + HBaseTextSerializerDeserializer.deserialize(inputSchema.getColumn(sortKeyIndexes[i]), + rowKeyFields[i])); + } + } + if (comparator.compare(dataRange.getStart(), endTuple) < 0) { + if (comparator.compare(dataRange.getStart(), previousTuple) >= 0) { + previousTuple = dataRange.getStart(); + } + tupleRanges.add(new TupleRange(sortSpecs, previousTuple, endTuple)); + previousTuple = endTuple; + } + } + + // Last region endkey is empty. Tajo ignores empty key, so endkey is replaced with max data value. + if (comparator.compare(dataRange.getEnd(), tupleRanges.get(tupleRanges.size() - 1).getStart()) >= 0) { + tupleRanges.get(tupleRanges.size() - 1).setEnd(dataRange.getEnd()); + } else { + tupleRanges.remove(tupleRanges.size() - 1); + } + return tupleRanges.toArray(new TupleRange[]{}); + } finally { + htable.close(); + } + } + + @Override + public Column[] getIndexColumns(TableDesc tableDesc) throws IOException { + List indexColumns = new ArrayList(); + + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); + + boolean[] isRowKeys = columnMapping.getIsRowKeyMappings(); + for (int i = 0; i < isRowKeys.length; i++) { + if (isRowKeys[i]) { + indexColumns.add(tableDesc.getSchema().getColumn(i)); + } + } + + return indexColumns.toArray(new Column[]{}); + } + + @Override + public StorageProperty getStorageProperty() { + StorageProperty storageProperty = new StorageProperty(); + storageProperty.setSortedInsert(true); + storageProperty.setSupportsInsertInto(true); + return storageProperty; + } + + public void verifyTableCreation(LogicalNode node) throws IOException { + if (node.getType() == NodeType.CREATE_TABLE) { + CreateTableNode cNode = (CreateTableNode)node; + if (!cNode.isExternal()) { + TableMeta tableMeta = new TableMeta(cNode.getStorageType(), cNode.getOptions()); + createTable(tableMeta, cNode.getTableSchema(), cNode.isExternal(), cNode.isIfNotExists()); + } + } + } + + @Override + public void queryFailed(LogicalNode node) throws IOException { + if (node.getType() == NodeType.CREATE_TABLE) { + CreateTableNode cNode = (CreateTableNode)node; + if (cNode.isExternal()) { + return; + } + TableMeta tableMeta = new TableMeta(cNode.getStorageType(), cNode.getOptions()); + HBaseAdmin hAdmin = new HBaseAdmin(getHBaseConfiguration(conf, tableMeta)); + + try { + HTableDescriptor hTableDesc = parseHTableDescriptor(tableMeta, cNode.getTableSchema()); + LOG.info("Delete table cause query failed:" + hTableDesc.getName()); + hAdmin.disableTable(hTableDesc.getName()); + hAdmin.deleteTable(hTableDesc.getName()); + } finally { + hAdmin.close(); + } + } + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java index 89ffb39eb8..bbc2ed4540 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java @@ -28,32 +28,30 @@ import java.io.IOException; import java.io.OutputStream; -public class HBaseTextSerializerDeserializer implements SerializerDeserializer { - - @Override - public int serialize(Column col, Datum datum, OutputStream out, byte[] nullCharacters) throws IOException { - return 0; - } - - @Override - public Datum deserialize(Column col, byte[] bytes, int offset, int length, byte[] nullCharacters) throws IOException { +public class HBaseTextSerializerDeserializer { + public static Datum deserialize(Column col, byte[] bytes) throws IOException { Datum datum; switch (col.getDataType().getType()) { case INT1: case INT2: - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt2((short) NumberUtil.parseInt(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : + DatumFactory.createInt2((short)NumberUtil.parseInt(bytes, 0, bytes.length)); break; case INT4: - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt4(NumberUtil.parseInt(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : + DatumFactory.createInt4(NumberUtil.parseInt(bytes, 0, bytes.length)); break; case INT8: - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(new String(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : + DatumFactory.createInt8(new String(bytes, 0, bytes.length)); break; case FLOAT4: - datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat4(new String(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : + DatumFactory.createFloat4(new String(bytes, 0, bytes.length)); break; case FLOAT8: - datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat8(NumberUtil.parseDouble(bytes, offset, length)); + datum = bytes == null ? NullDatum.get() : + DatumFactory.createFloat8(NumberUtil.parseDouble(bytes, 0, bytes.length)); break; case TEXT: datum = bytes == null ? NullDatum.get() : DatumFactory.createText(bytes); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java index 3a3bb57359..b10d423156 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.parquet; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.storage.StorageConstants; import parquet.hadoop.ParquetOutputFormat; import parquet.hadoop.metadata.CompressionCodecName; @@ -51,11 +52,11 @@ public class ParquetAppender extends FileAppender { * @param conf Configuration properties. * @param schema The table schema. * @param meta The table metadata. - * @param path The path of the Parquet file to write to. + * @param workDir The path of the Parquet file to write to. */ - public ParquetAppender(Configuration conf, Schema schema, TableMeta meta, - Path path) throws IOException { - super(conf, schema, meta, path); + public ParquetAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema, TableMeta meta, + Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); this.blockSize = Integer.parseInt( meta.getOption(ParquetOutputFormat.BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE)); this.pageSize = Integer.parseInt( diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java index 0e5c0e97b3..23815d9f2d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.SequenceFile.Metadata; import org.apache.hadoop.io.compress.*; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -710,8 +711,9 @@ public long getLength() throws IOException { return out.getPos(); } - public RCFileAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path) throws IOException { - super(conf, schema, meta, path); + public RCFileAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId, + final Schema schema, final TableMeta meta, final Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); RECORD_INTERVAL = conf.getInt(RECORD_INTERVAL_CONF_STR, RECORD_INTERVAL); COLUMNS_BUFFER_SIZE = conf.getInt(COLUMNS_BUFFER_SIZE_CONF_STR, COLUMNS_BUFFER_SIZE); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java index f5cef6285c..14e0f2691d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -73,8 +74,9 @@ public class SequenceFileAppender extends FileAppender { private Writable EMPTY_KEY; - public SequenceFileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException { - super(conf, schema, meta, path); + public SequenceFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); this.meta = meta; this.schema = schema; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java index a17fbf343b..0681c2cdb9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -47,8 +48,9 @@ public class TrevniAppender extends FileAppender { private TableStatistics stats = null; private boolean flushed = false; - public TrevniAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException { - super(conf, schema, meta, path); + public TrevniAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); } public void init() throws IOException { diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java new file mode 100644 index 0000000000..3e21a71ef9 --- /dev/null +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; +import org.apache.hadoop.mapreduce.*; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; + +import org.junit.Test; + +public class TestHBaseAppender { + @Test + public void testOutputPath() throws Exception { + Configuration taskConf = new Configuration(); + taskConf.set(FileOutputFormat.OUTDIR, "file:///tmp/tajo-babokim/test/q-1234"); + TaskAttemptContext context = new TaskAttemptContextImpl(taskConf, + new TaskAttemptID("200707121733", 1, TaskType.MAP, 2, 3)); + HFileOutputFormat2 hFileOutputFormat2 = new HFileOutputFormat2(); + try { + RecordWriter writer = hFileOutputFormat2.getRecordWriter(context); + ImmutableBytesWritable key = new ImmutableBytesWritable(); + key.set("aaa".getBytes()); + + KeyValue value = new KeyValue("aaa".getBytes(), "cf1".getBytes(), "cname".getBytes(), "value1".getBytes()); + writer.write(key, value); + writer.close(context); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestHBaseStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java similarity index 96% rename from tajo-storage/src/test/java/org/apache/tajo/storage/TestHBaseStorageManager.java rename to tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java index 268dd70c13..c9e7384178 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestHBaseStorageManager.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.storage; +package org.apache.tajo.storage.hbase; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -26,6 +26,7 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.hbase.HBaseStorageManager; import org.apache.tajo.util.Pair; import org.junit.Test; @@ -50,7 +51,7 @@ public void testGetIndexPredications() throws Exception { scanNode.setQual(evalNodeA); HBaseStorageManager storageManager = - (HBaseStorageManager)StorageManager.getStorageManager(new TajoConf(), StoreType.HBASE); + (HBaseStorageManager) StorageManager.getStorageManager(new TajoConf(), StoreType.HBASE); List> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); assertNotNull(indexEvals); assertEquals(1, indexEvals.size()); From 8f09273e73cc47a31ad9d585cd3f7e6ad75e840f Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Thu, 30 Oct 2014 22:39:20 +0900 Subject: [PATCH 07/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. All test cases pass. --- .../java/org/apache/tajo/cli/TajoCli.java | 2 +- .../physical/ColPartitionStoreExec.java | 3 +- .../planner/physical/StoreTableExec.java | 3 +- .../tajo/engine/query/QueryContext.java | 4 +- .../org/apache/tajo/master/GlobalEngine.java | 3 +- .../apache/tajo/master/querymaster/Query.java | 16 ++- .../java/org/apache/tajo/worker/Task.java | 10 ++ .../tajo/worker/TaskAttemptContext.java | 16 +-- .../planner/physical/TestPhysicalPlanner.java | 19 +-- .../tajo/engine/query/TestHBaseTable.java | 92 +++++++++------ .../master/querymaster/TestKillQuery.java | 110 +++++++++--------- .../rules/AddSortForInsertRewriter.java | 18 +++ .../java/org/apache/tajo/storage/CSVFile.java | 2 +- .../tajo/storage/FileStorageManager.java | 6 +- .../tajo/storage/hbase/TestHBaseAppender.java | 52 --------- .../hbase/TestHBaseStorageManager.java | 3 - 16 files changed, 183 insertions(+), 176 deletions(-) delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index 05b919c265..39e66de38f 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -27,6 +27,7 @@ import org.apache.commons.cli.*; import org.apache.tajo.*; +import org.apache.tajo.ipc.*; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; @@ -34,7 +35,6 @@ import org.apache.tajo.client.TajoHAClientUtil; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.util.FileUtil; import java.io.*; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 1827461058..c5df5f916a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -119,8 +119,7 @@ public ColPartitionStoreExec(TaskAttemptContext context, StoreTableNode plan, Ph public void init() throws IOException { super.init(); - storeTablePath = StorageManager.getFileStorageManager(context.getConf()) - .getAppenderFilePath(context.getTaskId(), context.getWorkDir()); + storeTablePath = context.getOutputPath(); FileSystem fs = storeTablePath.getFileSystem(context.getConf()); if (!fs.exists(storeTablePath.getParent())) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index 64d8c7be46..6e7786681a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -85,8 +85,7 @@ public void openNewFile(int suffixId) throws IOException { if (PlannerUtil.isFileStorageType(meta.getStoreType())) { String prevFile = null; - lastFileName = StorageManager.getFileStorageManager(context.getConf()).getAppenderFilePath( - context.getTaskId(), context.getQueryContext().getStagingDir()); + lastFileName = context.getOutputPath(); if (suffixId > 0) { prevFile = lastFileName.toString(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 77e3517e2c..488cae52be 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -68,8 +68,8 @@ public void setStagingDir(Path path) { } public Path getStagingDir() { - String strVal = get(QueryVars.STAGING_DIR); - return strVal != null ? new Path(strVal) : null; + String strVal = get(QueryVars.STAGING_DIR, ""); + return strVal != null && !strVal.isEmpty() ? new Path(strVal) : null; } /** diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index fb434ed3bf..461b5ddbf9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -354,7 +354,8 @@ private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode } TaskAttemptContext taskAttemptContext = - new TaskAttemptContext(queryContext, null, null, (CatalogProtos.FragmentProto[]) null, stagingResultDir); + new TaskAttemptContext(queryContext, null, null, (CatalogProtos.FragmentProto[]) null, stagingDir); + taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000")); EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); StoreTableExec exec = new StoreTableExec(taskAttemptContext, insertNode, evalExprExec); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index bfd3bce260..3802060f99 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -386,12 +386,16 @@ public QueryState transition(Query query, QueryEvent queryEvent) { } if (finalState != QueryState.QUERY_SUCCEEDED) { SubQuery lastStage = query.getSubQuery(subQueryEvent.getExecutionBlockId()); - StoreType storeType = lastStage.getTableMeta().getStoreType(); - LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); - try { - StorageManager.getStorageManager(query.systemConf, storeType).queryFailed(rootNode.getChild()); - } catch (IOException e) { - LOG.warn(query.getId() + ", failed processing cleanup storage when query failed:" + e.getMessage(), e); + if (lastStage != null && lastStage.getTableMeta() != null) { + StoreType storeType = lastStage.getTableMeta().getStoreType(); + if (storeType != null) { + LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); + try { + StorageManager.getStorageManager(query.systemConf, storeType).queryFailed(rootNode.getChild()); + } catch (IOException e) { + LOG.warn(query.getId() + ", failed processing cleanup storage when query failed:" + e.getMessage(), e); + } + } } } query.eventHandler.handle(new QueryMasterQueryCompletedEvent(query.getId())); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 2bf0d76d2f..71e6522fd8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -53,6 +53,7 @@ import org.apache.tajo.pullserver.retriever.FileChunk; import org.apache.tajo.rpc.NullCallback; import org.apache.tajo.storage.HashShuffleAppenderManager; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; @@ -154,6 +155,11 @@ public Task(String taskRunnerId, this.finalSchema = PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()); this.sortComp = new TupleComparator(finalSchema, sortNode.getSortKeys()); } + } else { + Path outFilePath = StorageManager.getFileStorageManager(systemConf).getAppenderFilePath( + taskId, queryContext.getStagingDir()); + LOG.info("Output File Path: " + outFilePath); + context.setOutputPath(outFilePath); } this.localChunks = Collections.synchronizedList(new ArrayList()); @@ -485,6 +491,10 @@ public TaskHistory createTaskHistory() { taskHistory = new TaskHistory(getTaskId(), getStatus(), context.getProgress(), startTime, finishTime, reloadInputStats()); + if (context.getOutputPath() != null) { + taskHistory.setOutputPath(context.getOutputPath().toString()); + } + if (context.getWorkDir() != null) { taskHistory.setWorkingPath(context.getWorkDir().toString()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 6e7a1cb98a..99976d8e44 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -72,7 +72,7 @@ public class TaskAttemptContext { private File fetchIn; private boolean stopped = false; private boolean interQuery = false; -// private Path outputPath; + private Path outputPath; private DataChannel dataChannel; private Enforcer enforcer; private QueryContext queryContext; @@ -198,13 +198,13 @@ public void setInterQuery() { this.interQuery = true; } -// public void setOutputPath(Path outputPath) { -// this.outputPath = outputPath; -// } -// -// public Path getOutputPath() { -// return this.outputPath; -// } + public void setOutputPath(Path outputPath) { + this.outputPath = outputPath; + } + + public Path getOutputPath() { + return this.outputPath; + } public boolean isInterQuery() { return this.interQuery; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 592b6fd3d9..5ac5125bc2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -435,12 +435,12 @@ public final void testStorePlan() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); + ctx.setOutputPath(new Path(workDir, "grouped1")); Expr context = analyzer.parse(CreateTableAsStmts[0]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); - TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.CSV); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); @@ -450,7 +450,7 @@ public final void testStorePlan() throws IOException, PlanningException { exec.close(); Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), - new Path(workDir, "grouped1")); + ctx.getOutputPath()); scanner.init(); Tuple tuple; int i = 0; @@ -486,7 +486,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); - Path outputPath = new Path(workDir, "maxOutput"); + ctx.setOutputPath(new Path(workDir, "maxOutput")); Expr context = analyzer.parse(CreateTableAsStmts[3]); @@ -502,13 +502,13 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann // checking the number of punctuated files int expectedFileNum = (int) (stats.getNumBytes() / (float) StorageUnit.MB); - FileSystem fs = outputPath.getFileSystem(conf); - FileStatus [] statuses = fs.listStatus(outputPath.getParent()); + FileSystem fs = ctx.getOutputPath().getFileSystem(conf); + FileStatus [] statuses = fs.listStatus(ctx.getOutputPath().getParent()); assertEquals(expectedFileNum, statuses.length); // checking the file contents long totalNum = 0; - for (FileStatus status : fs.listStatus(outputPath.getParent())) { + for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner( CatalogUtil.newTableMeta(StoreType.CSV), rootNode.getOutSchema(), @@ -532,6 +532,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); + ctx.setOutputPath(new Path(workDir, "grouped2")); Expr context = analyzer.parse(CreateTableAsStmts[1]); LogicalPlan plan = planner.createPlan(defaultContext, context); @@ -546,7 +547,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio exec.close(); Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), - new Path(workDir, "grouped2")); + ctx.getOutputPath()); scanner.init(); Tuple tuple; int i = 0; @@ -572,6 +573,7 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); + ctx.setOutputPath(new Path(workDir, "grouped3")); Expr context = analyzer.parse(CreateTableAsStmts[2]); LogicalPlan plan = planner.createPlan(defaultContext, context); @@ -598,6 +600,7 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); + ctx.setOutputPath(new Path(workDir, "grouped4")); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -621,6 +624,7 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); + ctx.setOutputPath(new Path(workDir, "grouped5")); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -709,6 +713,7 @@ public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, // Preparing task context TaskAttemptContext ctx = new TaskAttemptContext(queryContext, id, new FileFragment[] { frags[0] }, workDir); + ctx.setOutputPath(new Path(workDir, "part-01-000000")); // SortBasedColumnPartitionStoreExec will be chosen by default. ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(CreateTableAsStmts[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index bd1857364a..170c1d5f53 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -134,7 +134,14 @@ public void testCreateHBaseTable() throws Exception { assertEquals("col2", hColumns[0].getNameAsString()); assertEquals("col3", hColumns[1].getNameAsString()); - executeString("DROP TABLE hbase_mapped_table1 PURGE"); + executeString("DROP TABLE hbase_mapped_table1 PURGE").close(); + + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); + try { + assertFalse(hAdmin.tableExists("hbase_table")); + } finally { + hAdmin.close(); + } } @Test @@ -156,7 +163,7 @@ public void testCreateNotExistsExternalHBaseTable() throws Exception { @Test public void testCreateExternalHBaseTable() throws Exception { - HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table_not_purge")); hTableDesc.addFamily(new HColumnDescriptor("col1")); hTableDesc.addFamily(new HColumnDescriptor("col2")); hTableDesc.addFamily(new HColumnDescriptor("col3")); @@ -167,13 +174,22 @@ public void testCreateExternalHBaseTable() throws Exception { assertNotNull(zkPort); executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + "USING hbase WITH ('table'='external_hbase_table_not_purge', 'columns'=':key,col1:a,col2:,col3:b', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); assertTableExists("external_hbase_mapped_table"); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + executeString("DROP TABLE external_hbase_mapped_table").close(); + + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); + try { + assertTrue(hAdmin.tableExists("external_hbase_table_not_purge")); + hAdmin.disableTable("external_hbase_table_not_purge"); + hAdmin.deleteTable("external_hbase_table_not_purge"); + } finally { + hAdmin.close(); + } } @Test @@ -213,7 +229,7 @@ public void testSimpleSelectQuery() throws Exception { ResultSet res = executeString("select * from external_hbase_mapped_table where rk > '20'"); assertResultSet(res); cleanupQuery(res); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); } finally { htable.close(); } @@ -270,7 +286,7 @@ public void testBinaryMappedQuery() throws Exception { assertEquals(expected, resultSetToString(res)); res.close(); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); } finally { htable.close(); } @@ -308,7 +324,7 @@ public void testRowFieldSelectQuery() throws Exception { ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 > 'field1-20'"); assertResultSet(res); cleanupQuery(res); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); } finally { htable.close(); } @@ -320,18 +336,18 @@ public void testIndexPredication() throws Exception { String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); assertNotNull(zkPort); - executeString("CREATE TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + "'hbase.split.rowkeys'='010,040,060,080', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); - assertTableExists("external_hbase_mapped_table"); + assertTableExists("hbase_mapped_table"); HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); - hAdmin.tableExists("external_hbase_table"); + hAdmin.tableExists("hbase_table"); - HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); try { org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); assertEquals(5, keys.getFirst().length); @@ -347,7 +363,7 @@ public void testIndexPredication() throws Exception { htable.put(put); } - TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "external_hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); // where rk >= '020' and rk <= '055' ScanNode scanNode = new ScanNode(1); @@ -360,7 +376,7 @@ public void testIndexPredication() throws Exception { scanNode.setQual(evalNodeA); StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE); - List fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); + List fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(2, fragments.size()); HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); @@ -377,7 +393,7 @@ public void testIndexPredication() throws Exception { new ConstEval(new TextDatum("075"))); EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); scanNode.setQual(evalNodeB); - fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); + fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(3, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); assertEquals("020", new String(fragment1.getStartRow())); @@ -400,7 +416,7 @@ public void testIndexPredication() throws Exception { EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); - fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); + fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(3, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); @@ -423,7 +439,7 @@ public void testIndexPredication() throws Exception { evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); - fragments = storageManager.getSplits("external_hbase_mapped_table", tableDesc, scanNode); + fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(2, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); @@ -434,10 +450,10 @@ public void testIndexPredication() throws Exception { assertEquals("040", new String(fragment2.getStartRow())); assertEquals("059", new String(fragment2.getStopRow())); - ResultSet res = executeString("select * from external_hbase_mapped_table where rk >= '020' and rk <= '055'"); + ResultSet res = executeString("select * from hbase_mapped_table where rk >= '020' and rk <= '055'"); assertResultSet(res); res.close(); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { htable.close(); hAdmin.close(); @@ -450,19 +466,19 @@ public void testNonForwardQuery() throws Exception { String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); assertNotNull(zkPort); - executeString("CREATE TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + "'hbase.split.rowkeys'='010,040,060,080', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); - assertTableExists("external_hbase_mapped_table"); + assertTableExists("hbase_mapped_table"); HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); HTable htable = null; try { - hAdmin.tableExists("external_hbase_table"); - htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + hAdmin.tableExists("hbase_table"); + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); assertEquals(5, keys.getFirst().length); @@ -477,10 +493,10 @@ public void testNonForwardQuery() throws Exception { htable.put(put); } - ResultSet res = executeString("select * from external_hbase_mapped_table"); + ResultSet res = executeString("select * from hbase_mapped_table"); assertResultSet(res); res.close(); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { hAdmin.close(); if (htable == null) { @@ -495,19 +511,19 @@ public void testJoin() throws Exception { String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); assertNotNull(zkPort); - executeString("CREATE TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 int8) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int8) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " + "'hbase.split.rowkeys'='010,040,060,080', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); - assertTableExists("external_hbase_mapped_table"); + assertTableExists("hbase_mapped_table"); HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); HTable htable = null; try { - hAdmin.tableExists("external_hbase_table"); - htable = new HTable(testingCluster.getHBaseUtil().getConf(), "external_hbase_table"); + hAdmin.tableExists("hbase_table"); + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); assertEquals(5, keys.getFirst().length); @@ -523,11 +539,11 @@ public void testJoin() throws Exception { } ResultSet res = executeString("select a.rk, a.col1, a.col2, a.col3, b.l_orderkey, b.l_linestatus " + - "from external_hbase_mapped_table a " + + "from hbase_mapped_table a " + "join default.lineitem b on a.col3 = b.l_orderkey"); assertResultSet(res); res.close(); - executeString("DROP TABLE external_hbase_mapped_table PURGE"); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { hAdmin.close(); if (htable != null) { @@ -568,6 +584,8 @@ public void testInsertInto() throws Exception { new byte[][]{null, Bytes.toBytes("col1"), Bytes.toBytes("col2"), Bytes.toBytes("col3")}, new byte[][]{null, Bytes.toBytes("a"), null, Bytes.toBytes("b")}, new boolean[]{false, false, false, true}, tableDesc.getSchema())); + + executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { if (scanner != null) { scanner.close(); @@ -626,6 +644,9 @@ public void testInsertIntoMultiRegion() throws Exception { new byte[][]{null, Bytes.toBytes("col1")}, new byte[][]{null, Bytes.toBytes("a")}, new boolean[]{false, false}, tableDesc.getSchema())); + + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { if (scanner != null) { scanner.close(); @@ -671,6 +692,8 @@ public void testInsertIntoRowField() throws Exception { new byte[][]{null, Bytes.toBytes("col1"), Bytes.toBytes("col2"), Bytes.toBytes("col3")}, new byte[][]{null, Bytes.toBytes("a"), Bytes.toBytes(""), Bytes.toBytes("b")}, new boolean[]{false, false, false, false}, tableDesc.getSchema())); + + executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { if (scanner != null) { scanner.close(); @@ -729,6 +752,9 @@ public void testCATS() throws Exception { new byte[][]{null, Bytes.toBytes("col1")}, new byte[][]{null, Bytes.toBytes("a")}, new boolean[]{false, false}, tableDesc.getSchema())); + + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { if (scanner != null) { scanner.close(); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java index 37ee402b79..f715407f2a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java @@ -50,59 +50,59 @@ public static void setUp() throws Exception { conf = cluster.getConfiguration(); } - @Test - public final void testKillQueryFromInitState() throws Exception { - SQLAnalyzer analyzer = new SQLAnalyzer(); - QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(conf); - Session session = LocalTajoTestingUtility.createDummySession(); - CatalogService catalog = cluster.getMaster().getCatalog(); - String query = "select l_orderkey from lineitem group by l_orderkey"; - - LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); - Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(defaultContext, expr); - - optimizer.optimize(plan); - - QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(conf); - MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); - GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); - globalPlanner.build(masterPlan); - - QueryMaster qm = cluster.getTajoWorkers().get(0).getWorkerContext().getQueryMaster(); - QueryMasterTask queryMasterTask = new QueryMasterTask(qm.getContext(), - queryId, session, defaultContext, expr.toJson(), plan.getRootBlock().getRoot().toJson()); - - queryMasterTask.init(conf); - queryMasterTask.getQueryTaskContext().getDispatcher().start(); - queryMasterTask.startQuery(); - - try{ - cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_RUNNING, 2); - } finally { - assertEquals(TajoProtos.QueryState.QUERY_RUNNING, queryMasterTask.getQuery().getSynchronizedState()); - } - - SubQuery subQuery = queryMasterTask.getQuery().getSubQueries().iterator().next(); - assertNotNull(subQuery); - - try{ - cluster.waitForSubQueryState(subQuery, SubQueryState.INITED, 2); - } finally { - assertEquals(SubQueryState.INITED, subQuery.getSynchronizedState()); - } - - // fire kill event - Query q = queryMasterTask.getQuery(); - q.handle(new QueryEvent(queryId, QueryEventType.KILL)); - - try{ - cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 10); - } finally { - assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState()); - } - queryMasterTask.stop(); - } +// @Test +// public final void testKillQueryFromInitState() throws Exception { +// SQLAnalyzer analyzer = new SQLAnalyzer(); +// QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(conf); +// Session session = LocalTajoTestingUtility.createDummySession(); +// CatalogService catalog = cluster.getMaster().getCatalog(); +// String query = "select l_orderkey from lineitem group by l_orderkey"; +// +// LogicalPlanner planner = new LogicalPlanner(catalog); +// LogicalOptimizer optimizer = new LogicalOptimizer(conf); +// Expr expr = analyzer.parse(query); +// LogicalPlan plan = planner.createPlan(defaultContext, expr); +// +// optimizer.optimize(plan); +// +// QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); +// QueryContext queryContext = new QueryContext(conf); +// MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); +// GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); +// globalPlanner.build(masterPlan); +// +// QueryMaster qm = cluster.getTajoWorkers().get(0).getWorkerContext().getQueryMaster(); +// QueryMasterTask queryMasterTask = new QueryMasterTask(qm.getContext(), +// queryId, session, defaultContext, expr.toJson(), plan.getRootBlock().getRoot().toJson()); +// +// queryMasterTask.init(conf); +// queryMasterTask.getQueryTaskContext().getDispatcher().start(); +// queryMasterTask.startQuery(); +// +// try{ +// cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_RUNNING, 2); +// } finally { +// assertEquals(TajoProtos.QueryState.QUERY_RUNNING, queryMasterTask.getQuery().getSynchronizedState()); +// } +// +// SubQuery subQuery = queryMasterTask.getQuery().getSubQueries().iterator().next(); +// assertNotNull(subQuery); +// +// try{ +// cluster.waitForSubQueryState(subQuery, SubQueryState.INITED, 2); +// } finally { +// assertEquals(SubQueryState.INITED, subQuery.getSynchronizedState()); +// } +// +// // fire kill event +// Query q = queryMasterTask.getQuery(); +// q.handle(new QueryEvent(queryId, QueryEventType.KILL)); +// +// try{ +// cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 10); +// } finally { +// assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState()); +// } +// queryMasterTask.stop(); +// } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java index fd831c7a0d..702b9c79de 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.plan.rewrite.rules; import org.apache.tajo.OverridableConf; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java index c4687273f9..04bcf71947 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java @@ -99,7 +99,7 @@ public CSVAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId, @Override public void init() throws IOException { if (!fs.exists(path.getParent())) { - throw new FileNotFoundException(path.toString()); + throw new FileNotFoundException(path.getParent().toString()); } //determine the intermediate file type diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 5e54bb7344..60aab12a7f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -823,7 +823,7 @@ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId fina Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); Path finalOutputDir; - if (queryContext.get(QueryVars.OUTPUT_TABLE_PATH) != null) { + if (!queryContext.get(QueryVars.OUTPUT_TABLE_PATH, "").isEmpty()) { finalOutputDir = new Path(queryContext.get(QueryVars.OUTPUT_TABLE_PATH)); try { FileSystem fs = stagingResultDir.getFileSystem(conf); @@ -837,7 +837,7 @@ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId fina boolean committed = false; Path oldTableDir = new Path(stagingDir, TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); - if (queryContext.get(QueryVars.OUTPUT_PARTITIONS) != null) { + if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { // This is a map for existing non-leaf directory to rename. A key is current directory and a value is // renaming directory. Map renameDirs = TUtil.newHashMap(); @@ -910,7 +910,7 @@ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId fina fmt.setGroupingUsed(false); fmt.setMinimumIntegerDigits(3); - if (queryContext.get(QueryVars.OUTPUT_PARTITIONS) != null) { + if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { if (eachFile.isFile()) { LOG.warn("Partition table can't have file in a staging dir: " + eachFile.getPath()); diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java deleted file mode 100644 index 3e21a71ef9..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseAppender.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; -import org.apache.hadoop.mapreduce.*; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; - -import org.junit.Test; - -public class TestHBaseAppender { - @Test - public void testOutputPath() throws Exception { - Configuration taskConf = new Configuration(); - taskConf.set(FileOutputFormat.OUTDIR, "file:///tmp/tajo-babokim/test/q-1234"); - TaskAttemptContext context = new TaskAttemptContextImpl(taskConf, - new TaskAttemptID("200707121733", 1, TaskType.MAP, 2, 3)); - HFileOutputFormat2 hFileOutputFormat2 = new HFileOutputFormat2(); - try { - RecordWriter writer = hFileOutputFormat2.getRecordWriter(context); - ImmutableBytesWritable key = new ImmutableBytesWritable(); - key.set("aaa".getBytes()); - - KeyValue value = new KeyValue("aaa".getBytes(), "cf1".getBytes(), "cname".getBytes(), "value1".getBytes()); - writer.write(key, value); - writer.close(context); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } -} diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java index c9e7384178..2a0ef33dc0 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java @@ -27,12 +27,9 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.hbase.HBaseStorageManager; import org.apache.tajo.util.Pair; import org.junit.Test; -import java.util.Arrays; -import java.util.Comparator; import java.util.List; import java.util.Set; From 76fb0a06451ecf9e254d3b5d85dafa9a9d248da6 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Sat, 1 Nov 2014 00:40:19 +0900 Subject: [PATCH 08/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Add data type verification --- .../org/apache/tajo/master/GlobalEngine.java | 23 +- .../tajo/engine/query/TestHBaseTable.java | 6 +- .../TestHBaseTable/testNonForwardQuery.result | 200 +++++++++--------- .../tajo/storage/FileStorageManager.java | 2 +- .../apache/tajo/storage/StorageManager.java | 7 +- .../tajo/storage/hbase/HBaseLazyTuple.java | 27 ++- .../storage/hbase/HBaseStorageManager.java | 19 +- 7 files changed, 173 insertions(+), 111 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 461b5ddbf9..a5335b1093 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -56,7 +56,6 @@ import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.verifier.LogicalPlanVerifier; import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; @@ -304,7 +303,7 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, if (!storageProperty.isSupportsInsertInto()) { throw new VerifyException("Inserting into non-file storage is not supported."); } - sm.verifyTableCreation(rootNode.getChild()); + sm.beforeCATS(rootNode.getChild()); } context.getSystemMetrics().counter("Query", "numDMLQuery").inc(); hookManager.doHooks(queryContext, plan); @@ -531,9 +530,29 @@ private LogicalPlan createLogicalPlan(QueryContext queryContext, Expr expression throw new VerifyException(sb.toString()); } + verifyInsertTableSchema(queryContext, state, plan); return plan; } + private void verifyInsertTableSchema(QueryContext queryContext, VerificationState state, LogicalPlan plan) { + StoreType storeType = PlannerUtil.getStoreType(plan); + if (storeType != null) { + // CATS or INSERT + String tableName = PlannerUtil.getStoreTableName(plan); + TableDesc tableDesc = catalog.getTableDesc(tableName); + + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + Schema outSchema = rootNode.getChild().getOutSchema(); + + try { + StorageManager.getStorageManager(queryContext.getConf(), storeType) + .verifyInsertTableSchema(tableDesc, outSchema); + } catch (Throwable t) { + state.addVerification(t.getMessage()); + } + } + } + /** * Alter a given table */ diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 170c1d5f53..61abc887d5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -466,8 +466,8 @@ public void testNonForwardQuery() throws Exception { String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); assertNotNull(zkPort); - executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + - "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:#b', " + "'hbase.split.rowkeys'='010,040,060,080', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); @@ -489,7 +489,7 @@ public void testNonForwardQuery() throws Exception { put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col3".getBytes(), "".getBytes(), Bytes.toBytes(i)); htable.put(put); } diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result index 799dff5aea..4f5fd8f2ef 100644 --- a/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result @@ -1,102 +1,102 @@ rk,col1,col2,col3 ------------------------------- -000,a-0,{"k1":"k1-0", "k2":"k2-0"},b-0 -001,a-1,{"k1":"k1-1", "k2":"k2-1"},b-1 -002,a-2,{"k1":"k1-2", "k2":"k2-2"},b-2 -003,a-3,{"k1":"k1-3", "k2":"k2-3"},b-3 -004,a-4,{"k1":"k1-4", "k2":"k2-4"},b-4 -005,a-5,{"k1":"k1-5", "k2":"k2-5"},b-5 -006,a-6,{"k1":"k1-6", "k2":"k2-6"},b-6 -007,a-7,{"k1":"k1-7", "k2":"k2-7"},b-7 -008,a-8,{"k1":"k1-8", "k2":"k2-8"},b-8 -009,a-9,{"k1":"k1-9", "k2":"k2-9"},b-9 -010,a-10,{"k1":"k1-10", "k2":"k2-10"},b-10 -011,a-11,{"k1":"k1-11", "k2":"k2-11"},b-11 -012,a-12,{"k1":"k1-12", "k2":"k2-12"},b-12 -013,a-13,{"k1":"k1-13", "k2":"k2-13"},b-13 -014,a-14,{"k1":"k1-14", "k2":"k2-14"},b-14 -015,a-15,{"k1":"k1-15", "k2":"k2-15"},b-15 -016,a-16,{"k1":"k1-16", "k2":"k2-16"},b-16 -017,a-17,{"k1":"k1-17", "k2":"k2-17"},b-17 -018,a-18,{"k1":"k1-18", "k2":"k2-18"},b-18 -019,a-19,{"k1":"k1-19", "k2":"k2-19"},b-19 -020,a-20,{"k1":"k1-20", "k2":"k2-20"},b-20 -021,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21 -022,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22 -023,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23 -024,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24 -025,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25 -026,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26 -027,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27 -028,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28 -029,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29 -030,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30 -031,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31 -032,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32 -033,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33 -034,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34 -035,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35 -036,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36 -037,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37 -038,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38 -039,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39 -040,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40 -041,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41 -042,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42 -043,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43 -044,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44 -045,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45 -046,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46 -047,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47 -048,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48 -049,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49 -050,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50 -051,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51 -052,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52 -053,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53 -054,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54 -055,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55 -056,a-56,{"k1":"k1-56", "k2":"k2-56"},b-56 -057,a-57,{"k1":"k1-57", "k2":"k2-57"},b-57 -058,a-58,{"k1":"k1-58", "k2":"k2-58"},b-58 -059,a-59,{"k1":"k1-59", "k2":"k2-59"},b-59 -060,a-60,{"k1":"k1-60", "k2":"k2-60"},b-60 -061,a-61,{"k1":"k1-61", "k2":"k2-61"},b-61 -062,a-62,{"k1":"k1-62", "k2":"k2-62"},b-62 -063,a-63,{"k1":"k1-63", "k2":"k2-63"},b-63 -064,a-64,{"k1":"k1-64", "k2":"k2-64"},b-64 -065,a-65,{"k1":"k1-65", "k2":"k2-65"},b-65 -066,a-66,{"k1":"k1-66", "k2":"k2-66"},b-66 -067,a-67,{"k1":"k1-67", "k2":"k2-67"},b-67 -068,a-68,{"k1":"k1-68", "k2":"k2-68"},b-68 -069,a-69,{"k1":"k1-69", "k2":"k2-69"},b-69 -070,a-70,{"k1":"k1-70", "k2":"k2-70"},b-70 -071,a-71,{"k1":"k1-71", "k2":"k2-71"},b-71 -072,a-72,{"k1":"k1-72", "k2":"k2-72"},b-72 -073,a-73,{"k1":"k1-73", "k2":"k2-73"},b-73 -074,a-74,{"k1":"k1-74", "k2":"k2-74"},b-74 -075,a-75,{"k1":"k1-75", "k2":"k2-75"},b-75 -076,a-76,{"k1":"k1-76", "k2":"k2-76"},b-76 -077,a-77,{"k1":"k1-77", "k2":"k2-77"},b-77 -078,a-78,{"k1":"k1-78", "k2":"k2-78"},b-78 -079,a-79,{"k1":"k1-79", "k2":"k2-79"},b-79 -080,a-80,{"k1":"k1-80", "k2":"k2-80"},b-80 -081,a-81,{"k1":"k1-81", "k2":"k2-81"},b-81 -082,a-82,{"k1":"k1-82", "k2":"k2-82"},b-82 -083,a-83,{"k1":"k1-83", "k2":"k2-83"},b-83 -084,a-84,{"k1":"k1-84", "k2":"k2-84"},b-84 -085,a-85,{"k1":"k1-85", "k2":"k2-85"},b-85 -086,a-86,{"k1":"k1-86", "k2":"k2-86"},b-86 -087,a-87,{"k1":"k1-87", "k2":"k2-87"},b-87 -088,a-88,{"k1":"k1-88", "k2":"k2-88"},b-88 -089,a-89,{"k1":"k1-89", "k2":"k2-89"},b-89 -090,a-90,{"k1":"k1-90", "k2":"k2-90"},b-90 -091,a-91,{"k1":"k1-91", "k2":"k2-91"},b-91 -092,a-92,{"k1":"k1-92", "k2":"k2-92"},b-92 -093,a-93,{"k1":"k1-93", "k2":"k2-93"},b-93 -094,a-94,{"k1":"k1-94", "k2":"k2-94"},b-94 -095,a-95,{"k1":"k1-95", "k2":"k2-95"},b-95 -096,a-96,{"k1":"k1-96", "k2":"k2-96"},b-96 -097,a-97,{"k1":"k1-97", "k2":"k2-97"},b-97 -098,a-98,{"k1":"k1-98", "k2":"k2-98"},b-98 -099,a-99,{"k1":"k1-99", "k2":"k2-99"},b-99 \ No newline at end of file +000,a-0,{"k1":"k1-0", "k2":"k2-0"},0 +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1 +002,a-2,{"k1":"k1-2", "k2":"k2-2"},2 +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3 +004,a-4,{"k1":"k1-4", "k2":"k2-4"},4 +005,a-5,{"k1":"k1-5", "k2":"k2-5"},5 +006,a-6,{"k1":"k1-6", "k2":"k2-6"},6 +007,a-7,{"k1":"k1-7", "k2":"k2-7"},7 +008,a-8,{"k1":"k1-8", "k2":"k2-8"},8 +009,a-9,{"k1":"k1-9", "k2":"k2-9"},9 +010,a-10,{"k1":"k1-10", "k2":"k2-10"},10 +011,a-11,{"k1":"k1-11", "k2":"k2-11"},11 +012,a-12,{"k1":"k1-12", "k2":"k2-12"},12 +013,a-13,{"k1":"k1-13", "k2":"k2-13"},13 +014,a-14,{"k1":"k1-14", "k2":"k2-14"},14 +015,a-15,{"k1":"k1-15", "k2":"k2-15"},15 +016,a-16,{"k1":"k1-16", "k2":"k2-16"},16 +017,a-17,{"k1":"k1-17", "k2":"k2-17"},17 +018,a-18,{"k1":"k1-18", "k2":"k2-18"},18 +019,a-19,{"k1":"k1-19", "k2":"k2-19"},19 +020,a-20,{"k1":"k1-20", "k2":"k2-20"},20 +021,a-21,{"k1":"k1-21", "k2":"k2-21"},21 +022,a-22,{"k1":"k1-22", "k2":"k2-22"},22 +023,a-23,{"k1":"k1-23", "k2":"k2-23"},23 +024,a-24,{"k1":"k1-24", "k2":"k2-24"},24 +025,a-25,{"k1":"k1-25", "k2":"k2-25"},25 +026,a-26,{"k1":"k1-26", "k2":"k2-26"},26 +027,a-27,{"k1":"k1-27", "k2":"k2-27"},27 +028,a-28,{"k1":"k1-28", "k2":"k2-28"},28 +029,a-29,{"k1":"k1-29", "k2":"k2-29"},29 +030,a-30,{"k1":"k1-30", "k2":"k2-30"},30 +031,a-31,{"k1":"k1-31", "k2":"k2-31"},31 +032,a-32,{"k1":"k1-32", "k2":"k2-32"},32 +033,a-33,{"k1":"k1-33", "k2":"k2-33"},33 +034,a-34,{"k1":"k1-34", "k2":"k2-34"},34 +035,a-35,{"k1":"k1-35", "k2":"k2-35"},35 +036,a-36,{"k1":"k1-36", "k2":"k2-36"},36 +037,a-37,{"k1":"k1-37", "k2":"k2-37"},37 +038,a-38,{"k1":"k1-38", "k2":"k2-38"},38 +039,a-39,{"k1":"k1-39", "k2":"k2-39"},39 +040,a-40,{"k1":"k1-40", "k2":"k2-40"},40 +041,a-41,{"k1":"k1-41", "k2":"k2-41"},41 +042,a-42,{"k1":"k1-42", "k2":"k2-42"},42 +043,a-43,{"k1":"k1-43", "k2":"k2-43"},43 +044,a-44,{"k1":"k1-44", "k2":"k2-44"},44 +045,a-45,{"k1":"k1-45", "k2":"k2-45"},45 +046,a-46,{"k1":"k1-46", "k2":"k2-46"},46 +047,a-47,{"k1":"k1-47", "k2":"k2-47"},47 +048,a-48,{"k1":"k1-48", "k2":"k2-48"},48 +049,a-49,{"k1":"k1-49", "k2":"k2-49"},49 +050,a-50,{"k1":"k1-50", "k2":"k2-50"},50 +051,a-51,{"k1":"k1-51", "k2":"k2-51"},51 +052,a-52,{"k1":"k1-52", "k2":"k2-52"},52 +053,a-53,{"k1":"k1-53", "k2":"k2-53"},53 +054,a-54,{"k1":"k1-54", "k2":"k2-54"},54 +055,a-55,{"k1":"k1-55", "k2":"k2-55"},55 +056,a-56,{"k1":"k1-56", "k2":"k2-56"},56 +057,a-57,{"k1":"k1-57", "k2":"k2-57"},57 +058,a-58,{"k1":"k1-58", "k2":"k2-58"},58 +059,a-59,{"k1":"k1-59", "k2":"k2-59"},59 +060,a-60,{"k1":"k1-60", "k2":"k2-60"},60 +061,a-61,{"k1":"k1-61", "k2":"k2-61"},61 +062,a-62,{"k1":"k1-62", "k2":"k2-62"},62 +063,a-63,{"k1":"k1-63", "k2":"k2-63"},63 +064,a-64,{"k1":"k1-64", "k2":"k2-64"},64 +065,a-65,{"k1":"k1-65", "k2":"k2-65"},65 +066,a-66,{"k1":"k1-66", "k2":"k2-66"},66 +067,a-67,{"k1":"k1-67", "k2":"k2-67"},67 +068,a-68,{"k1":"k1-68", "k2":"k2-68"},68 +069,a-69,{"k1":"k1-69", "k2":"k2-69"},69 +070,a-70,{"k1":"k1-70", "k2":"k2-70"},70 +071,a-71,{"k1":"k1-71", "k2":"k2-71"},71 +072,a-72,{"k1":"k1-72", "k2":"k2-72"},72 +073,a-73,{"k1":"k1-73", "k2":"k2-73"},73 +074,a-74,{"k1":"k1-74", "k2":"k2-74"},74 +075,a-75,{"k1":"k1-75", "k2":"k2-75"},75 +076,a-76,{"k1":"k1-76", "k2":"k2-76"},76 +077,a-77,{"k1":"k1-77", "k2":"k2-77"},77 +078,a-78,{"k1":"k1-78", "k2":"k2-78"},78 +079,a-79,{"k1":"k1-79", "k2":"k2-79"},79 +080,a-80,{"k1":"k1-80", "k2":"k2-80"},80 +081,a-81,{"k1":"k1-81", "k2":"k2-81"},81 +082,a-82,{"k1":"k1-82", "k2":"k2-82"},82 +083,a-83,{"k1":"k1-83", "k2":"k2-83"},83 +084,a-84,{"k1":"k1-84", "k2":"k2-84"},84 +085,a-85,{"k1":"k1-85", "k2":"k2-85"},85 +086,a-86,{"k1":"k1-86", "k2":"k2-86"},86 +087,a-87,{"k1":"k1-87", "k2":"k2-87"},87 +088,a-88,{"k1":"k1-88", "k2":"k2-88"},88 +089,a-89,{"k1":"k1-89", "k2":"k2-89"},89 +090,a-90,{"k1":"k1-90", "k2":"k2-90"},90 +091,a-91,{"k1":"k1-91", "k2":"k2-91"},91 +092,a-92,{"k1":"k1-92", "k2":"k2-92"},92 +093,a-93,{"k1":"k1-93", "k2":"k2-93"},93 +094,a-94,{"k1":"k1-94", "k2":"k2-94"},94 +095,a-95,{"k1":"k1-95", "k2":"k2-95"},95 +096,a-96,{"k1":"k1-96", "k2":"k2-96"},96 +097,a-97,{"k1":"k1-97", "k2":"k2-97"},97 +098,a-98,{"k1":"k1-98", "k2":"k2-98"},98 +099,a-99,{"k1":"k1-99", "k2":"k2-99"},99 \ No newline at end of file diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 60aab12a7f..a79268ea79 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -1118,7 +1118,7 @@ public Column[] getIndexColumns(TableDesc tableDesc) throws IOException { } @Override - public void verifyTableCreation(LogicalNode node) throws IOException { + public void beforeCATS(LogicalNode node) throws IOException { } @Override diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index d85011f2e9..4bb8fd2dfd 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -167,11 +167,10 @@ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, T public abstract Column[] getIndexColumns(TableDesc tableDesc) throws IOException; /** - * * @param node * @throws IOException */ - public abstract void verifyTableCreation(LogicalNode node) throws IOException; + public abstract void beforeCATS(LogicalNode node) throws IOException; /** * @@ -396,4 +395,8 @@ public static long getFragmentLength(TajoConf conf, Fragment fragment) { return fragment.getLength(); } } + + public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { + // nothing to do + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java index 6e39fe5834..82e91543d6 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java @@ -132,9 +132,32 @@ public Datum get(int fieldId) { int count = 0; String delim = ""; + if (cfMap.size() == 0) { + values[fieldId] = NullDatum.get(); + return values[fieldId]; + } else if (cfMap.size() == 1) { + // If a column family is mapped without column name like "cf1:" and the number of cells is one, + // return value is flat format not json format. + NavigableMap.Entry entry = cfMap.entrySet().iterator().next(); + byte[] entryKey = entry.getKey(); + byte[] entryValue = entry.getValue(); + if (entryKey == null || entryKey.length == 0) { + try { + if (isBinaryColumns[fieldId]) { + values[fieldId] = HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue); + } else { + values[fieldId] = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue); + } + return values[fieldId]; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + } StringBuilder sb = new StringBuilder(); sb.append("{"); - for (NavigableMap.Entry entry: cfMap.entrySet()) { + for (NavigableMap.Entry entry : cfMap.entrySet()) { byte[] entryKey = entry.getKey(); byte[] entryValue = entry.getValue(); @@ -157,7 +180,7 @@ public Datum get(int fieldId) { if (count > 100) { break; } - } + } //end of for sb.append("}"); values[fieldId] = new TextDatum(sb.toString()); return values[fieldId]; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 73c990fbb9..e4be285fd7 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -986,7 +986,7 @@ public StorageProperty getStorageProperty() { return storageProperty; } - public void verifyTableCreation(LogicalNode node) throws IOException { + public void beforeCATS(LogicalNode node) throws IOException { if (node.getType() == NodeType.CREATE_TABLE) { CreateTableNode cNode = (CreateTableNode)node; if (!cNode.isExternal()) { @@ -1016,4 +1016,21 @@ public void queryFailed(LogicalNode node) throws IOException { } } } + + @Override + public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { + Schema tableSchema = tableDesc.getSchema(); + if (tableSchema.size() != outSchema.size()) { + throw new IOException("The number of table columns is different from SELECT columns"); + } + + for (int i = 0; i < tableSchema.size(); i++) { + if (!tableSchema.getColumn(i).getDataType().equals(outSchema.getColumn(i).getDataType())) { + throw new IOException(outSchema.getColumn(i).getQualifiedName() + + "(" + outSchema.getColumn(i).getDataType() + ")" + + " is different column type with " + tableSchema.getColumn(i).getSimpleName() + + "(" + tableSchema.getColumn(i).getDataType() + ")"); + } + } + } } From e8d50a05279517db8ee8e49eec5d5eb369d3a45b Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Sat, 1 Nov 2014 15:54:56 +0900 Subject: [PATCH 09/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Supporting binary split key --- .../org/apache/tajo/master/GlobalEngine.java | 26 +-- .../tajo/engine/query/TestHBaseTable.java | 167 +++++++++++++++++- .../testInsertIntoBinaryMultiRegion.result | 100 +++++++++++ .../testInsertIntoMultiRegion2.result | 100 +++++++++++ .../HBaseBinarySerializerDeserializer.java | 12 +- .../storage/hbase/HBaseStorageManager.java | 142 +++++++++------ .../HBaseTextSerializerDeserializer.java | 10 +- 7 files changed, 478 insertions(+), 79 deletions(-) create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index a5335b1093..174bbe07fd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -521,6 +521,7 @@ private LogicalPlan createLogicalPlan(QueryContext queryContext, Expr expression LOG.info("============================================="); annotatedPlanVerifier.verify(queryContext, state, plan); + verifyInsertTableSchema(queryContext, state, plan); if (!state.verified()) { StringBuilder sb = new StringBuilder(); @@ -530,25 +531,26 @@ private LogicalPlan createLogicalPlan(QueryContext queryContext, Expr expression throw new VerifyException(sb.toString()); } - verifyInsertTableSchema(queryContext, state, plan); return plan; } private void verifyInsertTableSchema(QueryContext queryContext, VerificationState state, LogicalPlan plan) { StoreType storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { - // CATS or INSERT - String tableName = PlannerUtil.getStoreTableName(plan); - TableDesc tableDesc = catalog.getTableDesc(tableName); - LogicalRootNode rootNode = plan.getRootBlock().getRoot(); - Schema outSchema = rootNode.getChild().getOutSchema(); - - try { - StorageManager.getStorageManager(queryContext.getConf(), storeType) - .verifyInsertTableSchema(tableDesc, outSchema); - } catch (Throwable t) { - state.addVerification(t.getMessage()); + if (rootNode.getChild().getType() == NodeType.INSERT) { + String tableName = PlannerUtil.getStoreTableName(plan); + TableDesc tableDesc = catalog.getTableDesc(tableName); + + InsertNode iNode = rootNode.getChild(); + Schema outSchema = iNode.getChild().getOutSchema(); + + try { + StorageManager.getStorageManager(queryContext.getConf(), storeType) + .verifyInsertTableSchema(tableDesc, outSchema); + } catch (Throwable t) { + state.addVerification(t.getMessage()); + } } } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 61abc887d5..c46582e550 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -567,7 +567,7 @@ public void testInsertInto() throws Exception { TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); executeString("insert into hbase_mapped_table " + - "select l_orderkey::text, l_shipdate, l_returnflag, l_suppkey from default.lineitem "); + "select l_orderkey::text, l_shipdate, l_returnflag, l_suppkey from default.lineitem ").close(); HTable htable = null; ResultScanner scanner = null; @@ -629,7 +629,7 @@ public void testInsertIntoMultiRegion() throws Exception { schema, tableOptions, datas.toArray(new String[]{}), 2); executeString("insert into hbase_mapped_table " + - "select id, name from base_table "); + "select id, name from base_table ").close(); HTable htable = null; ResultScanner scanner = null; @@ -658,6 +658,167 @@ public void testInsertIntoMultiRegion() throws Exception { } } + @Test + public void testInsertIntoMultiRegion2() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " + + "'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", Type.TEXT); + schema.addColumn("name", Type.TEXT); + List datas = new ArrayList(); + for (int i = 99; i >= 0; i--) { + datas.add(i + "|value" + i); + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("insert into hbase_mapped_table " + + "select id, name from base_table ").close(); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col1")}, + new byte[][]{null, Bytes.toBytes("a")}, + new boolean[]{false, false}, tableDesc.getSchema())); + + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + } finally { + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + + @Test + public void testInsertIntoBinaryMultiRegion() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk int4, col1 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key#b,col1:a', " + + "'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", Type.INT4); + schema.addColumn("name", Type.TEXT); + List datas = new ArrayList(); + for (int i = 99; i >= 0; i--) { + datas.add(i + "|value" + i); + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("insert into hbase_mapped_table " + + "select id, name from base_table ").close(); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col1")}, + new byte[][]{null, Bytes.toBytes("a")}, + new boolean[]{true, false}, tableDesc.getSchema())); + + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + } finally { + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + + @Test + public void testInsertIntoDifferentType() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " + + "'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", Type.INT4); + schema.addColumn("name", Type.TEXT); + List datas = new ArrayList(); + for (int i = 99; i >= 0; i--) { + datas.add(i + "|value" + i); + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + try { + executeString("insert into hbase_mapped_table " + + "select id, name from base_table ").close(); + fail("If inserting data type different with target table data type, should throw exception"); + } catch (Exception e) { + assertTrue(e.getMessage().indexOf("VerifyException") >= 0); + } finally { + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + } + } + @Test public void testInsertIntoRowField() throws Exception { String hostName = InetAddress.getLocalHost().getHostName(); @@ -774,7 +935,7 @@ private String resultSetToString(ResultScanner scanner, Result result = null; while ( (result = scanner.next()) != null ) { if (binaries[0]) { - sb.append(Bytes.toLong(result.getRow())); + sb.append(HBaseBinarySerializerDeserializer.deserialize(schema.getColumn(0), result.getRow()).asChar()); } else { sb.append(new String(result.getRow())); } diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result new file mode 100644 index 0000000000..c55873a141 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result @@ -0,0 +1,100 @@ +0, value0 +1, value1 +2, value2 +3, value3 +4, value4 +5, value5 +6, value6 +7, value7 +8, value8 +9, value9 +1, value10 +1, value11 +1, value12 +1, value13 +1, value14 +1, value15 +1, value16 +1, value17 +1, value18 +1, value19 +2, value20 +2, value21 +2, value22 +2, value23 +2, value24 +2, value25 +2, value26 +2, value27 +2, value28 +2, value29 +3, value30 +3, value31 +3, value32 +3, value33 +3, value34 +3, value35 +3, value36 +3, value37 +3, value38 +3, value39 +4, value40 +4, value41 +4, value42 +4, value43 +4, value44 +4, value45 +4, value46 +4, value47 +4, value48 +4, value49 +5, value50 +5, value51 +5, value52 +5, value53 +5, value54 +5, value55 +5, value56 +5, value57 +5, value58 +5, value59 +6, value60 +6, value61 +6, value62 +6, value63 +6, value64 +6, value65 +6, value66 +6, value67 +6, value68 +6, value69 +7, value70 +7, value71 +7, value72 +7, value73 +7, value74 +7, value75 +7, value76 +7, value77 +7, value78 +7, value79 +8, value80 +8, value81 +8, value82 +8, value83 +8, value84 +8, value85 +8, value86 +8, value87 +8, value88 +8, value89 +9, value90 +9, value91 +9, value92 +9, value93 +9, value94 +9, value95 +9, value96 +9, value97 +9, value98 +9, value99 diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result new file mode 100644 index 0000000000..be11042b0e --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result @@ -0,0 +1,100 @@ +0, value0 +1, value1 +10, value10 +11, value11 +12, value12 +13, value13 +14, value14 +15, value15 +16, value16 +17, value17 +18, value18 +19, value19 +2, value2 +20, value20 +21, value21 +22, value22 +23, value23 +24, value24 +25, value25 +26, value26 +27, value27 +28, value28 +29, value29 +3, value3 +30, value30 +31, value31 +32, value32 +33, value33 +34, value34 +35, value35 +36, value36 +37, value37 +38, value38 +39, value39 +4, value4 +40, value40 +41, value41 +42, value42 +43, value43 +44, value44 +45, value45 +46, value46 +47, value47 +48, value48 +49, value49 +5, value5 +50, value50 +51, value51 +52, value52 +53, value53 +54, value54 +55, value55 +56, value56 +57, value57 +58, value58 +59, value59 +6, value6 +60, value60 +61, value61 +62, value62 +63, value63 +64, value64 +65, value65 +66, value66 +67, value67 +68, value68 +69, value69 +7, value7 +70, value70 +71, value71 +72, value72 +73, value73 +74, value74 +75, value75 +76, value76 +77, value77 +78, value78 +79, value79 +8, value8 +80, value80 +81, value81 +82, value82 +83, value83 +84, value84 +85, value85 +86, value86 +87, value87 +88, value88 +89, value89 +9, value9 +90, value90 +91, value91 +92, value92 +93, value93 +94, value94 +95, value95 +96, value96 +97, value97 +98, value98 +99, value99 diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java index 39075b3522..3bfeea8c0f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java @@ -35,23 +35,23 @@ public static Datum deserialize(Column col, byte[] bytes) throws IOException { switch (col.getDataType().getType()) { case INT1: case INT2: - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt2(Bytes.toShort(bytes)); + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt2(Bytes.toShort(bytes)); break; case INT4: - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes)); + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes)); break; case INT8: if (bytes.length == 4) { - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes)); + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes)); } else { - datum = bytes == null ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes)); + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes)); } break; case FLOAT4: - datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat4(Bytes.toFloat(bytes)); + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createFloat4(Bytes.toFloat(bytes)); break; case FLOAT8: - datum = bytes == null ? NullDatum.get() : DatumFactory.createFloat8(Bytes.toDouble(bytes)); + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createFloat8(Bytes.toDouble(bytes)); break; case TEXT: datum = bytes == null ? NullDatum.get() : DatumFactory.createText(bytes); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index e4be285fd7..e69107014d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -42,6 +42,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.*; @@ -148,7 +149,7 @@ private void createTable(TableMeta tableMeta, Schema schema, // Creating hbase table HTableDescriptor hTableDescriptor = parseHTableDescriptor(tableMeta, schema); - byte[][] splitKeys = getSplitKeys(conf, tableMeta); + byte[][] splitKeys = getSplitKeys(conf, schema, tableMeta); if (splitKeys == null) { hAdmin.createTable(hTableDescriptor); } else { @@ -160,7 +161,7 @@ private void createTable(TableMeta tableMeta, Schema schema, } } - private byte[][] getSplitKeys(TajoConf conf, TableMeta meta) throws IOException { + private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) throws IOException { String splitRowKeys = meta.getOption(META_SPLIT_ROW_KEYS_KEY, ""); String splitRowKeysFile = meta.getOption(META_SPLIT_ROW_KEYS_FILE_KEY, ""); @@ -169,11 +170,37 @@ private byte[][] getSplitKeys(TajoConf conf, TableMeta meta) throws IOException return null; } + ColumnMapping columnMapping = new ColumnMapping(schema, meta); + boolean[] isBinaryColumns = columnMapping.getIsBinaryColumns(); + boolean[] isRowKeys = columnMapping.getIsRowKeyMappings(); + + boolean rowkeyBinary = false; + int numRowKeys = 0; + Column rowKeyColumn = null; + for (int i = 0; i < isBinaryColumns.length; i++) { + if (isBinaryColumns[i] && isRowKeys[i]) { + rowkeyBinary = true; + } + if (isRowKeys[i]) { + numRowKeys++; + rowKeyColumn = schema.getColumn(i); + } + } + + if (rowkeyBinary && numRowKeys > 1) { + throw new IOException("If rowkey is mapped to multi column and a rowkey is binary, " + + "Multiple region for creation is not support."); + } + if (splitRowKeys != null && !splitRowKeys.isEmpty()) { String[] splitKeyTokens = splitRowKeys.split(","); byte[][] splitKeys = new byte[splitKeyTokens.length][]; for (int i = 0; i < splitKeyTokens.length; i++) { - splitKeys[i] = Bytes.toBytes(splitKeyTokens[i]); + if (numRowKeys == 1 && rowkeyBinary) { + splitKeys[i] = HBaseBinarySerializerDeserializer.serialize(rowKeyColumn, new TextDatum(splitKeyTokens[i])); + } else { + splitKeys[i] = HBaseTextSerializerDeserializer.serialize(rowKeyColumn, new TextDatum(splitKeyTokens[i])); + } } return splitKeys; } @@ -210,6 +237,11 @@ private byte[][] getSplitKeys(TajoConf conf, TableMeta meta) throws IOException int index = 0; for (String eachKey: splitKeySet) { splitKeys[index++] = Bytes.toBytes(eachKey); + if (numRowKeys == 1 && rowkeyBinary) { + splitKeys[index++] = HBaseBinarySerializerDeserializer.serialize(rowKeyColumn, new TextDatum(eachKey)); + } else { + splitKeys[index++] = HBaseTextSerializerDeserializer.serialize(rowKeyColumn, new TextDatum(eachKey)); + } } return splitKeys; @@ -898,67 +930,71 @@ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId fina public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException { - int[] sortKeyIndexes = new int[sortSpecs.length]; - for (int i = 0; i < sortSpecs.length; i++) { - sortKeyIndexes[i] = inputSchema.getColumnId(sortSpecs[i].getSortKey().getQualifiedName()); - } - - ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); - Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); - - HTable htable = new HTable(hbaseConf, columnMapping.getHbaseTableName()); try { - byte[][] endKeys = htable.getEndKeys(); - if (endKeys.length == 1) { - return new TupleRange[]{dataRange}; - } - List tupleRanges = new ArrayList(endKeys.length); - - TupleComparator comparator = new TupleComparator(inputSchema, sortSpecs); - Tuple previousTuple = new VTuple(sortSpecs.length); + int[] sortKeyIndexes = new int[sortSpecs.length]; for (int i = 0; i < sortSpecs.length; i++) { - previousTuple.put(i, NullDatum.get()); + sortKeyIndexes[i] = inputSchema.getColumnId(sortSpecs[i].getSortKey().getQualifiedName()); } - for (byte[] eachEndKey: endKeys) { - Tuple endTuple = new VTuple(sortSpecs.length); - byte[][] rowKeyFields; - if (sortSpecs.length > 1) { - rowKeyFields = BytesUtils.splitPreserveAllTokens(eachEndKey, columnMapping.getRowKeyDelimiter()); - } else { - rowKeyFields = new byte[1][]; - rowKeyFields[0] = eachEndKey; + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); + Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); + + HTable htable = new HTable(hbaseConf, columnMapping.getHbaseTableName()); + try { + byte[][] endKeys = htable.getEndKeys(); + if (endKeys.length == 1) { + return new TupleRange[]{dataRange}; } + List tupleRanges = new ArrayList(endKeys.length); + TupleComparator comparator = new TupleComparator(inputSchema, sortSpecs); + Tuple previousTuple = new VTuple(sortSpecs.length); for (int i = 0; i < sortSpecs.length; i++) { - if (columnMapping.getIsBinaryColumns()[sortKeyIndexes[i]]) { - endTuple.put(i, - HBaseBinarySerializerDeserializer.deserialize(inputSchema.getColumn(sortKeyIndexes[i]), - rowKeyFields[i])); + previousTuple.put(i, NullDatum.get()); + } + + for (byte[] eachEndKey : endKeys) { + Tuple endTuple = new VTuple(sortSpecs.length); + byte[][] rowKeyFields; + if (sortSpecs.length > 1) { + rowKeyFields = BytesUtils.splitPreserveAllTokens(eachEndKey, columnMapping.getRowKeyDelimiter()); } else { - endTuple.put(i, - HBaseTextSerializerDeserializer.deserialize(inputSchema.getColumn(sortKeyIndexes[i]), - rowKeyFields[i])); + rowKeyFields = new byte[1][]; + rowKeyFields[0] = eachEndKey; } - } - if (comparator.compare(dataRange.getStart(), endTuple) < 0) { - if (comparator.compare(dataRange.getStart(), previousTuple) >= 0) { - previousTuple = dataRange.getStart(); + + for (int i = 0; i < sortSpecs.length; i++) { + if (columnMapping.getIsBinaryColumns()[sortKeyIndexes[i]]) { + endTuple.put(i, + HBaseBinarySerializerDeserializer.deserialize(inputSchema.getColumn(sortKeyIndexes[i]), + rowKeyFields[i])); + } else { + endTuple.put(i, + HBaseTextSerializerDeserializer.deserialize(inputSchema.getColumn(sortKeyIndexes[i]), + rowKeyFields[i])); + } + } + if (comparator.compare(dataRange.getStart(), endTuple) < 0) { + if (comparator.compare(dataRange.getStart(), previousTuple) >= 0) { + previousTuple = dataRange.getStart(); + } + tupleRanges.add(new TupleRange(sortSpecs, previousTuple, endTuple)); + previousTuple = endTuple; } - tupleRanges.add(new TupleRange(sortSpecs, previousTuple, endTuple)); - previousTuple = endTuple; } - } - // Last region endkey is empty. Tajo ignores empty key, so endkey is replaced with max data value. - if (comparator.compare(dataRange.getEnd(), tupleRanges.get(tupleRanges.size() - 1).getStart()) >= 0) { - tupleRanges.get(tupleRanges.size() - 1).setEnd(dataRange.getEnd()); - } else { - tupleRanges.remove(tupleRanges.size() - 1); + // Last region endkey is empty. Tajo ignores empty key, so endkey is replaced with max data value. + if (comparator.compare(dataRange.getEnd(), tupleRanges.get(tupleRanges.size() - 1).getStart()) >= 0) { + tupleRanges.get(tupleRanges.size() - 1).setEnd(dataRange.getEnd()); + } else { + tupleRanges.remove(tupleRanges.size() - 1); + } + return tupleRanges.toArray(new TupleRange[]{}); + } finally { + htable.close(); } - return tupleRanges.toArray(new TupleRange[]{}); - } finally { - htable.close(); + } catch (Throwable t) { + throw new IOException(t.getMessage(), t); } } @@ -1027,9 +1063,9 @@ public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throw for (int i = 0; i < tableSchema.size(); i++) { if (!tableSchema.getColumn(i).getDataType().equals(outSchema.getColumn(i).getDataType())) { throw new IOException(outSchema.getColumn(i).getQualifiedName() + - "(" + outSchema.getColumn(i).getDataType() + ")" + + "(" + outSchema.getColumn(i).getDataType().getType() + ")" + " is different column type with " + tableSchema.getColumn(i).getSimpleName() + - "(" + tableSchema.getColumn(i).getDataType() + ")"); + "(" + tableSchema.getColumn(i).getDataType().getType() + ")"); } } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java index bbc2ed4540..52b76f6504 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java @@ -34,23 +34,23 @@ public static Datum deserialize(Column col, byte[] bytes) throws IOException { switch (col.getDataType().getType()) { case INT1: case INT2: - datum = bytes == null ? NullDatum.get() : + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt2((short)NumberUtil.parseInt(bytes, 0, bytes.length)); break; case INT4: - datum = bytes == null ? NullDatum.get() : + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt4(NumberUtil.parseInt(bytes, 0, bytes.length)); break; case INT8: - datum = bytes == null ? NullDatum.get() : + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(new String(bytes, 0, bytes.length)); break; case FLOAT4: - datum = bytes == null ? NullDatum.get() : + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createFloat4(new String(bytes, 0, bytes.length)); break; case FLOAT8: - datum = bytes == null ? NullDatum.get() : + datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createFloat8(NumberUtil.parseDouble(bytes, 0, bytes.length)); break; case TEXT: From 37942472653febd82f1a7f55c0cfc79b53d780f4 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Sat, 1 Nov 2014 17:03:55 +0900 Subject: [PATCH 10/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Fix HBaseAppender unsorted bug. --- .../tajo/storage/hbase/HBaseAppender.java | 51 ++++++++++++++----- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java index 1fb2c61b69..31b1b619e6 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java @@ -21,7 +21,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.hbase.Cell; @@ -50,6 +49,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.TreeSet; public class HBaseAppender implements Appender { private static final Log LOG = LogFactory.getLog(HBaseAppender.class); @@ -138,6 +138,9 @@ public void init() throws IOException { long totalNumBytes = 0; ByteArrayOutputStream bout = new ByteArrayOutputStream(); ImmutableBytesWritable keyWritable = new ImmutableBytesWritable(); + boolean first = true; + TreeSet kvSet = new TreeSet(KeyValue.COMPARATOR); + @Override public void addTuple(Tuple tuple) throws IOException { Datum datum; @@ -168,6 +171,24 @@ public void addTuple(Tuple tuple) throws IOException { rowkey = HBaseTextSerializerDeserializer.serialize(schema.getColumn(index), datum); } } + if (!first && !Bytes.equals(keyWritable.get(), 0, keyWritable.getLength(), rowkey, 0, rowkey.length)) { + try { + for (KeyValue kv : kvSet) { + writer.write(keyWritable, kv); + totalNumBytes += keyWritable.getLength() + keyWritable.getLength(); + } + kvSet.clear(); + // Statistical section + if (enabledStats) { + stats.incrementRow(); + } + } catch (InterruptedException e) { + LOG.error(e.getMessage(), e); + } + } + + first = false; + keyWritable.set(rowkey); for (int i = 0; i < columnNum; i++) { if (isRowKeyMappings[i]) { @@ -181,17 +202,7 @@ public void addTuple(Tuple tuple) throws IOException { value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(i), datum); } KeyValue keyValue = new KeyValue(rowkey, mappingColumnFamilies[i][0], mappingColumnFamilies[i][1], value); - - try { - writer.write(keyWritable, keyValue); - } catch (InterruptedException e) { - LOG.error(e.getMessage(), e); - } - totalNumBytes += keyWritable.getLength() + keyValue.getLength(); - } - // Statistical section - if (enabledStats) { - stats.incrementRow(); + kvSet.add(keyValue); } } @@ -208,6 +219,22 @@ public long getEstimatedOutputSize() throws IOException { @Override public void close() throws IOException { + if (!kvSet.isEmpty()) { + try { + for (KeyValue kv : kvSet) { + writer.write(keyWritable, kv); + totalNumBytes += keyWritable.getLength() + keyWritable.getLength(); + } + kvSet.clear(); + // Statistical section + if (enabledStats) { + stats.incrementRow(); + } + } catch (InterruptedException e) { + LOG.error(e.getMessage(), e); + } + } + if (enabledStats) { stats.setNumBytes(totalNumBytes); } From b636faf5023ed6523077f46dd4583f9414209e39 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 00:46:55 +0900 Subject: [PATCH 11/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Support multiple rowkey field. --- .../main/java/org/apache/tajo/util/TUtil.java | 9 ++ .../apache/tajo/engine/utils/TupleUtil.java | 7 +- .../master/querymaster/QueryMasterTask.java | 11 +- .../tajo/engine/query/TestHBaseTable.java | 83 ++++++++++++++- ...InsertIntoMultiRegionMultiRowFields.result | 100 ++++++++++++++++++ .../apache/tajo/plan/logical/SortNode.java | 6 +- .../tajo/storage/FileStorageManager.java | 5 - .../java/org/apache/tajo/storage/RawFile.java | 1 - .../apache/tajo/storage/StorageManager.java | 13 +-- .../tajo/storage/fragment/FileFragment.java | 4 +- .../hbase}/AddSortForInsertRewriter.java | 30 ++++-- .../tajo/storage/hbase/ColumnMapping.java | 2 +- .../tajo/storage/hbase/HBaseAppender.java | 1 - .../storage/hbase/HBaseStorageManager.java | 51 +++++++-- 14 files changed, 277 insertions(+), 46 deletions(-) create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result rename {tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules => tajo-storage/src/main/java/org/apache/tajo/storage/hbase}/AddSortForInsertRewriter.java (74%) diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java index 0ceb2b2dbe..dcf5c4e701 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java @@ -119,6 +119,15 @@ public static List newList() { return new ArrayList(); } + public static List newIntList(int...items) { + List list = new ArrayList(); + for (int item: items) { + list.add(item); + } + + return list; + } + public static List newList(T...items) { List list = new ArrayList(); for (T t : items) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java index aeb4e05110..3bb1b5ba44 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java @@ -125,12 +125,13 @@ public static TupleRange columnStatToRange(SortSpec [] sortSpecs, Schema target, Tuple startTuple = new VTuple(target.size()); Tuple endTuple = new VTuple(target.size()); int i = 0; + int sortSpecIndex = 0; // In outer join, empty table could be searched. // As a result, min value and max value would be null. // So, we should put NullDatum for this case. for (Column col : target.getColumns()) { - if (sortSpecs[i].isAscending()) { + if (sortSpecs[sortSpecIndex].isAscending()) { if (statSet.get(col).getMinValue() != null) startTuple.put(i, statSet.get(col).getMinValue()); else @@ -164,6 +165,10 @@ public static TupleRange columnStatToRange(SortSpec [] sortSpecs, Schema target, else endTuple.put(i, DatumFactory.createNullDatum()); } + if (target.getColumns().size() == sortSpecs.length) { + // Not composite column sort + sortSpecIndex++; + } i++; } return new TupleRange(sortSpecs, startTuple, endTuple); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 6d6a9a2727..79800c2cdf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -42,7 +42,7 @@ import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.LogicalRootNode; import org.apache.tajo.plan.rewrite.RewriteRule; -import org.apache.tajo.plan.rewrite.rules.AddSortForInsertRewriter; +import org.apache.tajo.storage.hbase.AddSortForInsertRewriter; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.plan.logical.LogicalNode; @@ -375,8 +375,13 @@ public synchronized void startQuery() { if (tableDesc == null) { throw new VerifyException("Can't get table meta data from catalog: " + tableName); } - optimizer.addRuleAfterToJoinOpt(new AddSortForInsertRewriter( - getQueryTaskContext().getQueryContext(), tableDesc, sm.getIndexColumns(tableDesc))); + List storageSpecifiedRewriteRules = sm.getRewriteRules( + getQueryTaskContext().getQueryContext(), tableDesc); + if (storageSpecifiedRewriteRules != null) { + for (RewriteRule eachRule: storageSpecifiedRewriteRules) { + optimizer.addRuleAfterToJoinOpt(eachRule); + } + } } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index c46582e550..76aec08a46 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -97,7 +97,7 @@ public void testVerifyCreateHBaseTableRequiredMeta() throws Exception { fail("hbase table must have 'columns' meta"); } catch (Exception e) { - assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0); + assertTrue(e.getMessage().indexOf("'columns' property is required") >= 0); } try { @@ -161,6 +161,24 @@ public void testCreateNotExistsExternalHBaseTable() throws Exception { } } + @Test + public void testCreateRowFieldWithNonText() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + try { + executeString("CREATE TABLE hbase_mapped_table2 (rk1 int4, rk2 text, col3 text, col4 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'='0:key#b,1:key,col3:,col2:b', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + fail("Key field type should be TEXT type"); + } catch (Exception e) { + assertTrue(e.getMessage().indexOf("Key field type should be TEXT type") >= 0); + } + } + @Test public void testCreateExternalHBaseTable() throws Exception { HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table_not_purge")); @@ -718,6 +736,69 @@ public void testInsertIntoMultiRegion2() throws Exception { } } + @Test + public void testInsertIntoMultiRegionMultiRowFields() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk1 text, rk2 text, col1 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a', " + + "'hbase.split.rowkeys'='001,002,003,004,005,006,007,008,009', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id1", Type.TEXT); + schema.addColumn("id2", Type.TEXT); + schema.addColumn("name", Type.TEXT); + DecimalFormat df = new DecimalFormat("000"); + List datas = new ArrayList(); + for (int i = 99; i >= 0; i--) { + datas.add(df.format(i) + "|" + (i + 100) + "|value" + i); + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("insert into hbase_mapped_table " + + "select id1, id2, name from base_table ").close(); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, null, Bytes.toBytes("col1")}, + new byte[][]{null, null, Bytes.toBytes("a")}, + new boolean[]{false, false, false}, tableDesc.getSchema())); + + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + } finally { + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + @Test public void testInsertIntoBinaryMultiRegion() throws Exception { String hostName = InetAddress.getLocalHost().getHostName(); diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result new file mode 100644 index 0000000000..0b740c8092 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result @@ -0,0 +1,100 @@ +000_100, value0 +001_101, value1 +002_102, value2 +003_103, value3 +004_104, value4 +005_105, value5 +006_106, value6 +007_107, value7 +008_108, value8 +009_109, value9 +010_110, value10 +011_111, value11 +012_112, value12 +013_113, value13 +014_114, value14 +015_115, value15 +016_116, value16 +017_117, value17 +018_118, value18 +019_119, value19 +020_120, value20 +021_121, value21 +022_122, value22 +023_123, value23 +024_124, value24 +025_125, value25 +026_126, value26 +027_127, value27 +028_128, value28 +029_129, value29 +030_130, value30 +031_131, value31 +032_132, value32 +033_133, value33 +034_134, value34 +035_135, value35 +036_136, value36 +037_137, value37 +038_138, value38 +039_139, value39 +040_140, value40 +041_141, value41 +042_142, value42 +043_143, value43 +044_144, value44 +045_145, value45 +046_146, value46 +047_147, value47 +048_148, value48 +049_149, value49 +050_150, value50 +051_151, value51 +052_152, value52 +053_153, value53 +054_154, value54 +055_155, value55 +056_156, value56 +057_157, value57 +058_158, value58 +059_159, value59 +060_160, value60 +061_161, value61 +062_162, value62 +063_163, value63 +064_164, value64 +065_165, value65 +066_166, value66 +067_167, value67 +068_168, value68 +069_169, value69 +070_170, value70 +071_171, value71 +072_172, value72 +073_173, value73 +074_174, value74 +075_175, value75 +076_176, value76 +077_177, value77 +078_178, value78 +079_179, value79 +080_180, value80 +081_181, value81 +082_182, value82 +083_183, value83 +084_184, value84 +085_185, value85 +086_186, value86 +087_187, value87 +088_188, value88 +089_189, value89 +090_190, value90 +091_191, value91 +092_192, value92 +093_193, value93 +094_194, value94 +095_195, value95 +096_196, value96 +097_197, value97 +098_198, value98 +099_199, value99 diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java index 3c69a62950..48654432ae 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java @@ -71,8 +71,7 @@ public PlanString getPlanString() { PlanString planStr = new PlanString(this); StringBuilder sb = new StringBuilder("Sort Keys: "); for (int i = 0; i < sortKeys.length; i++) { - sb.append(sortKeys[i].getSortKey().getSimpleName()).append(" ") - .append(sortKeys[i].isAscending() ? "asc" : "desc"); + sb.append(sortKeys[i].toString()); if( i < sortKeys.length - 1) { sb.append(","); } @@ -84,8 +83,7 @@ public PlanString getPlanString() { public String toString() { StringBuilder sb = new StringBuilder("Sort [key= "); for (int i = 0; i < sortKeys.length; i++) { - sb.append(sortKeys[i].getSortKey().getQualifiedName()).append(" ") - .append(sortKeys[i].isAscending() ? "asc" : "desc"); + sb.append(sortKeys[i].toString()); if(i < sortKeys.length - 1) { sb.append(","); } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index a79268ea79..0555eb0b56 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -1112,11 +1112,6 @@ private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IO return tokens[0] + "-" + tokens[1] + "-" + tokens[2] + "-" + nf.format(seq); } - @Override - public Column[] getIndexColumns(TableDesc tableDesc) throws IOException { - return null; - } - @Override public void beforeCATS(LogicalNode node) throws IOException { } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java index a6fcbbda2f..003f8e5b44 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java @@ -79,7 +79,6 @@ public void init() throws IOException { } catch (IllegalArgumentException iae) { throw new IOException(iae); } - fis = new FileInputStream(file); channel = fis.getChannel(); fileLimit = fragment.getStartKey() + fragment.getLength(); // fileLimit is less than or equal to fileSize diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 4bb8fd2dfd..9c0f3f0e0e 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -33,10 +33,8 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.RewriteRule; -import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -159,13 +157,6 @@ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, T Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException; - /** - * - * @param tableDesc - * @throws IOException - */ - public abstract Column[] getIndexColumns(TableDesc tableDesc) throws IOException; - /** * @param node * @throws IOException @@ -399,4 +390,8 @@ public static long getFragmentLength(TajoConf conf, Fragment fragment) { public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { // nothing to do } + + public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { + return null; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index 970d1428ea..4a83dbf8d7 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -38,8 +38,8 @@ public class FileFragment implements Fragment, Comparable, Cloneable { @Expose private String tableName; // required @Expose private Path uri; // required - @Expose private Long startOffset; // required - @Expose private Long length; // required + @Expose public Long startOffset; // required + @Expose public Long length; // required private String[] hosts; // Datanode hostnames @Expose private int[] diskIds; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java similarity index 74% rename from tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java rename to tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java index 702b9c79de..e23c44df6f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AddSortForInsertRewriter.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.plan.rewrite.rules; +package org.apache.tajo.storage.hbase; import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; @@ -33,8 +33,9 @@ public class AddSortForInsertRewriter implements RewriteRule { private int[] sortColumnIndexes; - - public AddSortForInsertRewriter(OverridableConf queryContext, TableDesc tableDesc, Column[] sortColumns) { + private Column[] sortColumns; + public AddSortForInsertRewriter(TableDesc tableDesc, Column[] sortColumns) { + this.sortColumns = sortColumns; this.sortColumnIndexes = new int[sortColumns.length]; Schema tableSchema = tableDesc.getSchema(); @@ -51,28 +52,35 @@ public String getName() { @Override public boolean isEligible(LogicalPlan plan) { StoreType storeType = PlannerUtil.getStoreType(plan); - return storeType != null; } @Override public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); - UnaryNode topNode = rootNode.getChild(); + UnaryNode insertNode = rootNode.getChild(); + LogicalNode childNode = insertNode.getChild(); + Schema sortSchema = childNode.getOutSchema(); SortNode sortNode = plan.createNode(SortNode.class); sortNode.setSortRangeType(SortRangeType.USING_STORAGE_MANAGER); - sortNode.setInSchema(topNode.getInSchema()); - sortNode.setOutSchema(topNode.getInSchema()); + sortNode.setInSchema(sortSchema); + sortNode.setOutSchema(sortSchema); + + SortSpec[] sortSpecs = new SortSpec[sortColumns.length]; + int index = 0; - SortSpec[] sortSpecs = new SortSpec[sortColumnIndexes.length]; for (int i = 0; i < sortColumnIndexes.length; i++) { - sortSpecs[i] = new SortSpec(topNode.getInSchema().getColumn(sortColumnIndexes[i]), true, true); + Column sortColumn = sortSchema.getColumn(sortColumnIndexes[i]); + if (sortColumn == null) { + throw new PlanningException("Can't fine proper sort column:" + sortColumns[i]); + } + sortSpecs[index++] = new SortSpec(sortColumn, true, true); } sortNode.setSortSpecs(sortSpecs); - sortNode.setChild(topNode.getChild()); - topNode.setChild(sortNode); + sortNode.setChild(insertNode.getChild()); + insertNode.setChild(sortNode); plan.getRootBlock().registerNode(sortNode); return plan; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java index 1fe81be5f5..ebd9af0c0c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -68,7 +68,7 @@ public void init() throws IOException { List> hbaseColumnMappings = parseColumnMapping(tableMeta); if (hbaseColumnMappings == null || hbaseColumnMappings.isEmpty()) { - throw new IOException("columns property is required."); + throw new IOException("'columns' property is required."); } if (hbaseColumnMappings.size() != schema.getColumns().size()) { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java index 31b1b619e6..ed6088c5f7 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java @@ -146,7 +146,6 @@ public void addTuple(Tuple tuple) throws IOException { Datum datum; byte[] rowkey; - // make rowkey if (rowkeyColumnIndexes.length > 1) { bout.reset(); for (int i = 0; i < rowkeyColumnIndexes.length; i++) { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index e69107014d..367f051a17 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -39,12 +39,14 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.*; @@ -104,18 +106,34 @@ private void createTable(TableMeta tableMeta, Schema schema, } TableName hTableName = TableName.valueOf(hbaseTableName); - String columnMapping = tableMeta.getOption(META_COLUMNS_KEY, ""); - if (columnMapping != null && columnMapping.split(",").length > schema.size()) { + String mappedColumns = tableMeta.getOption(META_COLUMNS_KEY, ""); + if (mappedColumns != null && mappedColumns.split(",").length > schema.size()) { throw new IOException("Columns property has more entry than Tajo table columns"); } + ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta); + int numRowKeys = 0; + boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + for (int i = 0; i < isRowKeyMappings.length; i++) { + if (isRowKeyMappings[i]) { + numRowKeys++; + } + } + if (numRowKeys > 1) { + for (int i = 0; i < isRowKeyMappings.length; i++) { + if (schema.getColumn(i).getDataType().getType() != Type.TEXT) { + throw new IOException("Key field type should be TEXT type."); + } + } + } + Configuration hConf = getHBaseConfiguration(conf, tableMeta); HBaseAdmin hAdmin = new HBaseAdmin(hConf); try { if (isExternal) { // If tajo table is external table, only check validation. - if (columnMapping == null || columnMapping.isEmpty()) { + if (mappedColumns == null || mappedColumns.isEmpty()) { throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); } if (!hAdmin.tableExists(hTableName)) { @@ -128,7 +146,7 @@ private void createTable(TableMeta tableMeta, Schema schema, tableColumnFamilies.add(eachColumn.getNameAsString()); } - Collection mappingColumnFamilies = getColumnFamilies(columnMapping); + Collection mappingColumnFamilies = getColumnFamilies(mappedColumns); if (mappingColumnFamilies.isEmpty()) { throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); } @@ -957,7 +975,20 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc Tuple endTuple = new VTuple(sortSpecs.length); byte[][] rowKeyFields; if (sortSpecs.length > 1) { - rowKeyFields = BytesUtils.splitPreserveAllTokens(eachEndKey, columnMapping.getRowKeyDelimiter()); + byte[][] splitValues = BytesUtils.splitPreserveAllTokens(eachEndKey, columnMapping.getRowKeyDelimiter()); + if (splitValues.length == sortSpecs.length) { + rowKeyFields = splitValues; + } else { + rowKeyFields = new byte[sortSpecs.length][]; + for (int j = 0; j < sortSpecs.length; j++) { + if (j < splitValues.length) { + rowKeyFields[j] = splitValues[j]; + } else { + rowKeyFields[j] = null; + } + } + } + } else { rowKeyFields = new byte[1][]; rowKeyFields[0] = eachEndKey; @@ -994,12 +1025,18 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc htable.close(); } } catch (Throwable t) { + LOG.error(t.getMessage(), t); throw new IOException(t.getMessage(), t); } } - @Override - public Column[] getIndexColumns(TableDesc tableDesc) throws IOException { + public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { + List rules = new ArrayList(); + rules.add(new AddSortForInsertRewriter(tableDesc, getIndexColumns(tableDesc))); + return rules; + } + + private Column[] getIndexColumns(TableDesc tableDesc) throws IOException { List indexColumns = new ArrayList(); ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); From 79cea448833281ffb081d8f21aefa645b082db2f Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 01:17:38 +0900 Subject: [PATCH 12/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. TupleComparator error --- .../tajo/engine/query/TestHBaseTable.java | 7 ++++--- .../apache/tajo/storage/TupleComparator.java | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 76aec08a46..9f3389f5e1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -744,7 +744,7 @@ public void testInsertIntoMultiRegionMultiRowFields() throws Exception { executeString("CREATE TABLE hbase_mapped_table (rk1 text, rk2 text, col1 text) " + "USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a', " + - "'hbase.split.rowkeys'='001,002,003,004,005,006,007,008,009', " + + "'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " + "'hbase.rowkey.delimiter'='_', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); @@ -761,10 +761,11 @@ public void testInsertIntoMultiRegionMultiRowFields() throws Exception { schema.addColumn("id1", Type.TEXT); schema.addColumn("id2", Type.TEXT); schema.addColumn("name", Type.TEXT); - DecimalFormat df = new DecimalFormat("000"); + //DecimalFormat df = new DecimalFormat("000"); List datas = new ArrayList(); for (int i = 99; i >= 0; i--) { - datas.add(df.format(i) + "|" + (i + 100) + "|value" + i); +// datas.add(df.format(i) + "|" + (i + 100) + "|value" + i); + datas.add(i + "|" + (i + 100) + "|value" + i); } TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", schema, tableOptions, datas.toArray(new String[]{}), 2); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java index 084c105d2b..c3a5ed47c8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java @@ -88,6 +88,23 @@ public int compare(Tuple tuple1, Tuple tuple2) { Datum right = null; int compVal = 0; + String sb1 = ""; + String sb2 = ""; + for (int i = 0; i < sortKeyIds.length; i++) { + left = tuple1.get(sortKeyIds[i]); + right = tuple2.get(sortKeyIds[i]); + if (i > 0) { + sb1 += "_" + left.asChars(); + sb2 += "_" + right.asChars(); + } else { + sb1 += "_" + left.asChars(); + sb2 += "_" + right.asChars(); + } + } + if (1==1) { + return sb1.compareTo(sb2); + } + for (int i = 0; i < sortKeyIds.length; i++) { left = tuple1.get(sortKeyIds[i]); right = tuple2.get(sortKeyIds[i]); From 529d48c112099cb8d85311f32a74986a7e2019b8 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 01:20:28 +0900 Subject: [PATCH 13/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. TupleComparator error --- ...InsertIntoMultiRegionMultiRowFields.result | 199 +++++++++--------- 1 file changed, 99 insertions(+), 100 deletions(-) diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result index 0b740c8092..07f9999be3 100644 --- a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result @@ -1,100 +1,99 @@ -000_100, value0 -001_101, value1 -002_102, value2 -003_103, value3 -004_104, value4 -005_105, value5 -006_106, value6 -007_107, value7 -008_108, value8 -009_109, value9 -010_110, value10 -011_111, value11 -012_112, value12 -013_113, value13 -014_114, value14 -015_115, value15 -016_116, value16 -017_117, value17 -018_118, value18 -019_119, value19 -020_120, value20 -021_121, value21 -022_122, value22 -023_123, value23 -024_124, value24 -025_125, value25 -026_126, value26 -027_127, value27 -028_128, value28 -029_129, value29 -030_130, value30 -031_131, value31 -032_132, value32 -033_133, value33 -034_134, value34 -035_135, value35 -036_136, value36 -037_137, value37 -038_138, value38 -039_139, value39 -040_140, value40 -041_141, value41 -042_142, value42 -043_143, value43 -044_144, value44 -045_145, value45 -046_146, value46 -047_147, value47 -048_148, value48 -049_149, value49 -050_150, value50 -051_151, value51 -052_152, value52 -053_153, value53 -054_154, value54 -055_155, value55 -056_156, value56 -057_157, value57 -058_158, value58 -059_159, value59 -060_160, value60 -061_161, value61 -062_162, value62 -063_163, value63 -064_164, value64 -065_165, value65 -066_166, value66 -067_167, value67 -068_168, value68 -069_169, value69 -070_170, value70 -071_171, value71 -072_172, value72 -073_173, value73 -074_174, value74 -075_175, value75 -076_176, value76 -077_177, value77 -078_178, value78 -079_179, value79 -080_180, value80 -081_181, value81 -082_182, value82 -083_183, value83 -084_184, value84 -085_185, value85 -086_186, value86 -087_187, value87 -088_188, value88 -089_189, value89 -090_190, value90 -091_191, value91 -092_192, value92 -093_193, value93 -094_194, value94 -095_195, value95 -096_196, value96 -097_197, value97 -098_198, value98 -099_199, value99 +10_110, value10 +11_111, value11 +12_112, value12 +13_113, value13 +14_114, value14 +15_115, value15 +16_116, value16 +17_117, value17 +18_118, value18 +19_119, value19 +1_101, value1 +20_120, value20 +21_121, value21 +22_122, value22 +23_123, value23 +24_124, value24 +25_125, value25 +26_126, value26 +27_127, value27 +28_128, value28 +29_129, value29 +2_102, value2 +30_130, value30 +31_131, value31 +32_132, value32 +33_133, value33 +34_134, value34 +35_135, value35 +36_136, value36 +37_137, value37 +38_138, value38 +39_139, value39 +3_103, value3 +40_140, value40 +41_141, value41 +42_142, value42 +43_143, value43 +44_144, value44 +45_145, value45 +46_146, value46 +47_147, value47 +48_148, value48 +49_149, value49 +4_104, value4 +50_150, value50 +51_151, value51 +52_152, value52 +53_153, value53 +54_154, value54 +55_155, value55 +56_156, value56 +57_157, value57 +58_158, value58 +59_159, value59 +5_105, value5 +60_160, value60 +61_161, value61 +62_162, value62 +63_163, value63 +64_164, value64 +65_165, value65 +66_166, value66 +67_167, value67 +68_168, value68 +69_169, value69 +6_106, value6 +70_170, value70 +71_171, value71 +72_172, value72 +73_173, value73 +74_174, value74 +75_175, value75 +76_176, value76 +77_177, value77 +78_178, value78 +79_179, value79 +7_107, value7 +80_180, value80 +81_181, value81 +82_182, value82 +83_183, value83 +84_184, value84 +85_185, value85 +86_186, value86 +87_187, value87 +88_188, value88 +89_189, value89 +8_108, value8 +90_190, value90 +91_191, value91 +92_192, value92 +93_193, value93 +94_194, value94 +95_195, value95 +96_196, value96 +97_197, value97 +98_198, value98 +99_199, value99 +9_109, value9 From 6ec23170f16d3a740714bc909dc8949ad00217e7 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 15:20:17 +0900 Subject: [PATCH 14/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Implements HBasePutAppender --- .../physical/RangeShuffleFileWriteExec.java | 2 + .../planner/physical/StoreTableExec.java | 1 + .../tajo/engine/query/TestHBaseTable.java | 58 ++++++- ...InsertIntoMultiRegionMultiRowFields.result | 1 + .../testInsertIntoUsingPut.result | 3 + .../tajo/storage/FileStorageManager.java | 2 +- .../apache/tajo/storage/StorageManager.java | 3 +- .../storage/hbase/AbstractHBaseAppender.java | 159 ++++++++++++++++++ .../tajo/storage/hbase/HBasePutAppender.java | 107 ++++++++++++ .../storage/hbase/HBaseStorageManager.java | 43 ++--- ...{HBaseAppender.java => HFileAppender.java} | 112 ++---------- .../src/main/resources/storage-default.xml | 5 +- 12 files changed, 367 insertions(+), 129 deletions(-) create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java create mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java rename tajo-storage/src/main/java/org/apache/tajo/storage/hbase/{HBaseAppender.java => HFileAppender.java} (58%) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index ccdfc4db31..4efcfeaaa0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -27,6 +27,8 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.index.bst.BSTIndex; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index 6e7786681a..a5e0b5d370 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -100,6 +100,7 @@ public void openNewFile(int suffixId) throws IOException { } } else { appender = StorageManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender( + context.getQueryContext(), context.getTaskId(), meta, appenderSchema, context.getQueryContext().getStagingDir()); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 9f3389f5e1..a20be42da0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -41,6 +41,7 @@ import org.apache.tajo.storage.hbase.HBaseTextSerializerDeserializer; import org.apache.tajo.util.Bytes; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.TUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -49,10 +50,7 @@ import java.net.InetAddress; import java.sql.ResultSet; import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; @@ -1009,6 +1007,58 @@ public void testCATS() throws Exception { } } + @Test + public void testInsertIntoUsingPut() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int4) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + Map sessions = new HashMap(); + sessions.put(HBaseStorageManager.INSERT_PUT_MODE, "true"); + client.updateSessionVariables(sessions); + + HTable htable = null; + ResultScanner scanner = null; + try { + executeString("insert into hbase_mapped_table " + + "select l_orderkey::text, l_shipdate, l_returnflag, l_suppkey from default.lineitem ").close(); + + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col1")); + scan.addFamily(Bytes.toBytes("col2")); + scan.addFamily(Bytes.toBytes("col3")); + scanner = htable.getScanner(scan); + + // result is dirrerent with testInsertInto because l_orderkey is not unique. + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col1"), Bytes.toBytes("col2"), Bytes.toBytes("col3")}, + new byte[][]{null, Bytes.toBytes("a"), null, Bytes.toBytes("b")}, + new boolean[]{false, false, false, true}, tableDesc.getSchema())); + + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + } finally { + client.unsetSessionVariables(TUtil.newList(HBaseStorageManager.INSERT_PUT_MODE)); + + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + private String resultSetToString(ResultScanner scanner, byte[][] cfNames, byte[][] qualifiers, boolean[] binaries, diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result index 07f9999be3..1e6748a8da 100644 --- a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result @@ -1,3 +1,4 @@ +0_100, value0 10_110, value10 11_111, value11 12_112, value12 diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result new file mode 100644 index 0000000000..1c22960377 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result @@ -0,0 +1,3 @@ +1, 1996-04-12, {"": "N"}, 7311 +2, 1997-01-28, {"": "N"}, 1191 +3, 1993-11-09, {"": "R"}, 6540 diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 0555eb0b56..03d024821f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -153,7 +153,7 @@ public Path getTablePath(String tableName) { @VisibleForTesting public Appender getAppender(TableMeta meta, Schema schema, Path filePath) throws IOException { - return getAppender(null, meta, schema, filePath); + return getAppender(null, null, meta, schema, filePath); } public FileFragment[] split(String tableName) throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 9c0f3f0e0e..6736e49426 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -248,7 +248,8 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) thro return getScanner(meta, schema, fragment, schema); } - public Appender getAppender(QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) + public Appender getAppender(OverridableConf queryContext, + QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) throws IOException { Appender appender; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java new file mode 100644 index 0000000000..3510e48b2c --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.tajo.QueryUnitAttemptId; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TableStatistics; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.TUtil; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractHBaseAppender implements Appender { + protected Configuration conf; + protected Schema schema; + protected TableMeta meta; + protected QueryUnitAttemptId taskAttemptId; + protected Path stagingDir; + protected boolean inited = false; + + protected ColumnMapping columnMapping; + protected TableStatistics stats; + protected boolean enabledStats; + + protected int columnNum; + + protected byte[][][] mappingColumnFamilies; + protected boolean[] isBinaryColumns; + protected boolean[] isRowKeyMappings; + protected int[] rowKeyFieldIndexes; + protected int[] rowkeyColumnIndexes; + protected char rowKeyDelimiter; + + public AbstractHBaseAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path stagingDir) { + this.conf = conf; + this.schema = schema; + this.meta = meta; + this.stagingDir = stagingDir; + this.taskAttemptId = taskAttemptId; + } + + @Override + public void init() throws IOException { + if (inited) { + throw new IllegalStateException("FileAppender is already initialized."); + } + inited = true; + if (enabledStats) { + stats = new TableStatistics(this.schema); + } + columnMapping = new ColumnMapping(schema, meta); + + mappingColumnFamilies = columnMapping.getMappingColumns(); + + isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + List rowkeyColumnIndexList = new ArrayList(); + for (int i = 0; i < isRowKeyMappings.length; i++) { + if (isRowKeyMappings[i]) { + rowkeyColumnIndexList.add(i); + } + } + rowkeyColumnIndexes = TUtil.toArray(rowkeyColumnIndexList); + + isBinaryColumns = columnMapping.getIsBinaryColumns(); + rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); + rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); + + this.columnNum = schema.size(); + } + + private ByteArrayOutputStream bout = new ByteArrayOutputStream(); + + protected byte[] getRowKeyBytes(Tuple tuple) throws IOException { + Datum datum; + byte[] rowkey; + if (rowkeyColumnIndexes.length > 1) { + bout.reset(); + for (int i = 0; i < rowkeyColumnIndexes.length; i++) { + datum = tuple.get(rowkeyColumnIndexes[i]); + if (isBinaryColumns[rowkeyColumnIndexes[i]]) { + rowkey = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(rowkeyColumnIndexes[i]), datum); + } else { + rowkey = HBaseTextSerializerDeserializer.serialize(schema.getColumn(rowkeyColumnIndexes[i]), datum); + } + bout.write(rowkey); + if (i < rowkeyColumnIndexes.length - 1) { + bout.write(rowKeyDelimiter); + } + } + rowkey = bout.toByteArray(); + } else { + int index = rowkeyColumnIndexes[0]; + datum = tuple.get(index); + if (isBinaryColumns[index]) { + rowkey = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(index), datum); + } else { + rowkey = HBaseTextSerializerDeserializer.serialize(schema.getColumn(index), datum); + } + } + + return rowkey; + } + + protected KeyValue getKeyValue(Tuple tuple, int index, byte[] rowkey) throws IOException { + Datum datum = tuple.get(index); + byte[] value; + if (isBinaryColumns[index]) { + value = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(index), datum); + } else { + value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(index), datum); + } + return new KeyValue(rowkey, mappingColumnFamilies[index][0], mappingColumnFamilies[index][1], value); + } + + @Override + public void enableStats() { + enabledStats = true; + } + + @Override + public TableStats getStats() { + if (enabledStats) { + return stats.getTableStat(); + } else { + return null; + } + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java new file mode 100644 index 0000000000..3f73e5cba9 --- /dev/null +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Put; +import org.apache.tajo.QueryUnitAttemptId; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tuple; + +import java.io.IOException; + +public class HBasePutAppender extends AbstractHBaseAppender { + private HTableInterface htable; + + public HBasePutAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path stagingDir) { + super(conf, taskAttemptId, schema, meta, stagingDir); + } + + @Override + public void init() throws IOException { + super.init(); + + Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(conf, meta); + HConnection hconn = ((HBaseStorageManager) StorageManager.getStorageManager((TajoConf)conf, StoreType.HBASE)) + .getConnection(hbaseConf); + htable = hconn.getTable(columnMapping.getHbaseTableName()); + htable.setAutoFlushTo(false); + htable.setWriteBufferSize(5 * 1024 * 1024); + } + + long totalNumBytes; + @Override + public void addTuple(Tuple tuple) throws IOException { + byte[] rowkey = getRowKeyBytes(tuple); + totalNumBytes += rowkey.length; + Put put = new Put(rowkey); + for (int i = 0; i < columnNum; i++) { + if (isRowKeyMappings[i]) { + continue; + } + Datum datum = tuple.get(i); + byte[] value; + if (isBinaryColumns[i]) { + value = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(i), datum); + } else { + value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(i), datum); + } + put.add(mappingColumnFamilies[i][0], mappingColumnFamilies[i][1], value); + totalNumBytes += value.length; + } + + htable.put(put); + + if (enabledStats) { + stats.incrementRow(); + stats.setNumBytes(totalNumBytes); + } + } + + @Override + public void flush() throws IOException { + htable.flushCommits(); + } + + @Override + public long getEstimatedOutputSize() throws IOException { + return 0; + } + + @Override + public void close() throws IOException { + if (htable != null) { + htable.flushCommits(); + htable.close(); + } + if (enabledStats) { + stats.setNumBytes(totalNumBytes); + } + } +} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 367f051a17..d9ee5c3a66 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -32,17 +32,13 @@ import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.mapreduce.task.JobContextImpl; -import org.apache.tajo.ExecutionBlockId; -import org.apache.tajo.OverridableConf; -import org.apache.tajo.QueryVars; -import org.apache.tajo.TajoConstants; +import org.apache.tajo.*; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; @@ -67,6 +63,8 @@ public class HBaseStorageManager extends StorageManager { public static final String ROWKEY_COLUMN_MAPPING = "key"; public static final String META_ROWKEY_DELIMITER = "hbase.rowkey.delimiter"; + public static final String INSERT_PUT_MODE = "tajo.hbase.insert.put.mode"; + private Map connMap = new HashMap(); public HBaseStorageManager (StoreType storeType) { @@ -535,6 +533,17 @@ private byte[] serialize(ColumnMapping columnMapping, } } + @Override + public Appender getAppender(OverridableConf queryContext, + QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) + throws IOException { + if ("true".equalsIgnoreCase(queryContext.get(INSERT_PUT_MODE, "false"))) { + return new HBasePutAppender(conf, taskAttemptId, schema, meta, workDir); + } else { + return super.getAppender(queryContext, taskAttemptId, meta, schema, workDir); + } + } + @Override public List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { @@ -966,10 +975,7 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc List tupleRanges = new ArrayList(endKeys.length); TupleComparator comparator = new TupleComparator(inputSchema, sortSpecs); - Tuple previousTuple = new VTuple(sortSpecs.length); - for (int i = 0; i < sortSpecs.length; i++) { - previousTuple.put(i, NullDatum.get()); - } + Tuple previousTuple = dataRange.getStart(); for (byte[] eachEndKey : endKeys) { Tuple endTuple = new VTuple(sortSpecs.length); @@ -1005,13 +1011,8 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc rowKeyFields[i])); } } - if (comparator.compare(dataRange.getStart(), endTuple) < 0) { - if (comparator.compare(dataRange.getStart(), previousTuple) >= 0) { - previousTuple = dataRange.getStart(); - } - tupleRanges.add(new TupleRange(sortSpecs, previousTuple, endTuple)); - previousTuple = endTuple; - } + tupleRanges.add(new TupleRange(sortSpecs, previousTuple, endTuple)); + previousTuple = endTuple; } // Last region endkey is empty. Tajo ignores empty key, so endkey is replaced with max data value. @@ -1031,9 +1032,13 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc } public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { - List rules = new ArrayList(); - rules.add(new AddSortForInsertRewriter(tableDesc, getIndexColumns(tableDesc))); - return rules; + if ("false".equalsIgnoreCase(queryContext.get(INSERT_PUT_MODE, "false"))) { + List rules = new ArrayList(); + rules.add(new AddSortForInsertRewriter(tableDesc, getIndexColumns(tableDesc))); + return rules; + } else { + return null; + } } private Column[] getIndexColumns(TableDesc tableDesc) throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java similarity index 58% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java rename to tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java index ed6088c5f7..fb88f6cb00 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java @@ -51,68 +51,22 @@ import java.util.List; import java.util.TreeSet; -public class HBaseAppender implements Appender { - private static final Log LOG = LogFactory.getLog(HBaseAppender.class); +public class HFileAppender extends AbstractHBaseAppender { + private static final Log LOG = LogFactory.getLog(HFileAppender.class); private RecordWriter writer; - private Configuration conf; - private Schema schema; - private TableMeta meta; - private QueryUnitAttemptId taskAttemptId; - private Path stagingDir; - private boolean inited = false; private TaskAttemptContext writerContext; - private int columnNum; - private ColumnMapping columnMapping; - private TableStatistics stats; - private boolean enabledStats; - - private byte[][][] mappingColumnFamilies; - private boolean[] isBinaryColumns; - private boolean[] isRowKeyMappings; - private int[] rowKeyFieldIndexes; - private int[] rowkeyColumnIndexes; - private char rowKeyDelimiter; - private Path workingFilePath; private FileOutputCommitter committer; - public HBaseAppender (Configuration conf, QueryUnitAttemptId taskAttemptId, - Schema schema, TableMeta meta, Path stagingDir) { - this.conf = conf; - this.schema = schema; - this.meta = meta; - this.stagingDir = stagingDir; - this.taskAttemptId = taskAttemptId; + public HFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path stagingDir) { + super(conf, taskAttemptId, schema, meta, stagingDir); } @Override public void init() throws IOException { - if (inited) { - throw new IllegalStateException("FileAppender is already initialized."); - } - inited = true; - if (enabledStats) { - stats = new TableStatistics(this.schema); - } - - columnMapping = new ColumnMapping(schema, meta); - mappingColumnFamilies = columnMapping.getMappingColumns(); - - isRowKeyMappings = columnMapping.getIsRowKeyMappings(); - List rowkeyColumnIndexList = new ArrayList(); - for (int i = 0; i < isRowKeyMappings.length; i++) { - if (isRowKeyMappings[i]) { - rowkeyColumnIndexList.add(i); - } - } - rowkeyColumnIndexes = TUtil.toArray(rowkeyColumnIndexList); - - isBinaryColumns = columnMapping.getIsBinaryColumns(); - rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); - rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); - - this.columnNum = schema.size(); + super.init(); Configuration taskConf = new Configuration(); taskConf.set(FileOutputFormat.OUTDIR, stagingDir.toString()); @@ -136,7 +90,6 @@ public void init() throws IOException { } long totalNumBytes = 0; - ByteArrayOutputStream bout = new ByteArrayOutputStream(); ImmutableBytesWritable keyWritable = new ImmutableBytesWritable(); boolean first = true; TreeSet kvSet = new TreeSet(KeyValue.COMPARATOR); @@ -145,36 +98,13 @@ public void init() throws IOException { public void addTuple(Tuple tuple) throws IOException { Datum datum; - byte[] rowkey; - if (rowkeyColumnIndexes.length > 1) { - bout.reset(); - for (int i = 0; i < rowkeyColumnIndexes.length; i++) { - datum = tuple.get(rowkeyColumnIndexes[i]); - if (isBinaryColumns[rowkeyColumnIndexes[i]]) { - rowkey = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(rowkeyColumnIndexes[i]), datum); - } else { - rowkey = HBaseTextSerializerDeserializer.serialize(schema.getColumn(rowkeyColumnIndexes[i]), datum); - } - bout.write(rowkey); - if (i < rowkeyColumnIndexes.length - 1) { - bout.write(rowKeyDelimiter); - } - } - rowkey = bout.toByteArray(); - } else { - int index = rowkeyColumnIndexes[0]; - datum = tuple.get(index); - if (isBinaryColumns[index]) { - rowkey = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(index), datum); - } else { - rowkey = HBaseTextSerializerDeserializer.serialize(schema.getColumn(index), datum); - } - } + byte[] rowkey = getRowKeyBytes(tuple); + if (!first && !Bytes.equals(keyWritable.get(), 0, keyWritable.getLength(), rowkey, 0, rowkey.length)) { try { for (KeyValue kv : kvSet) { writer.write(keyWritable, kv); - totalNumBytes += keyWritable.getLength() + keyWritable.getLength(); + totalNumBytes += keyWritable.getLength() + kv.getLength(); } kvSet.clear(); // Statistical section @@ -193,15 +123,7 @@ public void addTuple(Tuple tuple) throws IOException { if (isRowKeyMappings[i]) { continue; } - datum = tuple.get(i); - byte[] value; - if (isBinaryColumns[i]) { - value = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(i), datum); - } else { - value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(i), datum); - } - KeyValue keyValue = new KeyValue(rowkey, mappingColumnFamilies[i][0], mappingColumnFamilies[i][1], value); - kvSet.add(keyValue); + kvSet.add(getKeyValue(tuple, i, rowkey)); } } @@ -245,18 +167,4 @@ public void close() throws IOException { } } } - - @Override - public void enableStats() { - enabledStats = true; - } - - @Override - public TableStats getStats() { - if (enabledStats) { - return stats.getTableStat(); - } else { - return null; - } - } } diff --git a/tajo-storage/src/main/resources/storage-default.xml b/tajo-storage/src/main/resources/storage-default.xml index d3096d034f..6efd3186e7 100644 --- a/tajo-storage/src/main/resources/storage-default.xml +++ b/tajo-storage/src/main/resources/storage-default.xml @@ -113,7 +113,7 @@ tajo.storage.appender-handler - csv,raw,rcfile,row,trevni,parquet,sequencefile,avro + csv,raw,rcfile,row,trevni,parquet,sequencefile,avro,hbase,hfile @@ -158,6 +158,7 @@ tajo.storage.appender-handler.hbase.class - org.apache.tajo.storage.hbase.HBaseAppender + org.apache.tajo.storage.hbase.HFileAppender + From 54974da8c1d0e0837035c9d92b87c4a30f13aa48 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 20:25:41 +0900 Subject: [PATCH 15/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Implements ColumnKey, Value mapping --- .../tajo/engine/query/TestHBaseTable.java | 187 ++++++++-- .../dataset/TestHBaseTable/splits.data | 0 .../testColumnKeyValueSelectQuery.result | 12 + .../testInsertIntoColumnKeyValue.result | 21 ++ .../storage/hbase/AbstractHBaseAppender.java | 81 ++++- .../tajo/storage/hbase/ColumnMapping.java | 178 ++++++---- .../tajo/storage/hbase/HBaseLazyTuple.java | 333 ------------------ .../tajo/storage/hbase/HBasePutAppender.java | 20 +- .../tajo/storage/hbase/HBaseScanner.java | 195 +++++++++- .../storage/hbase/HBaseStorageManager.java | 67 +--- .../tajo/storage/hbase/HFileAppender.java | 10 +- .../tajo/storage/hbase/TestColumnMapping.java | 95 +++++ 12 files changed, 713 insertions(+), 486 deletions(-) create mode 100644 tajo-core/src/test/resources/dataset/TestHBaseTable/splits.data create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java create mode 100644 tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index a20be42da0..e2d8e28e00 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -245,8 +245,8 @@ public void testSimpleSelectQuery() throws Exception { ResultSet res = executeString("select * from external_hbase_mapped_table where rk > '20'"); assertResultSet(res); cleanupQuery(res); - executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); htable.close(); } } @@ -290,20 +290,62 @@ public void testBinaryMappedQuery() throws Exception { res.close(); //Projection - res = executeString("select col3 from external_hbase_mapped_table where rk > 95"); + res = executeString("select col3, col2, rk from external_hbase_mapped_table where rk > 95"); - String expected = "col3\n" + + String expected = "col3,col2,rk\n" + "-------------------------------\n" + - "96\n" + - "97\n" + - "98\n" + - "99\n"; + "96,{\"k1\":\"k1-96\", \"k2\":\"k2-96\"},96\n" + + "97,{\"k1\":\"k1-97\", \"k2\":\"k2-97\"},97\n" + + "98,{\"k1\":\"k1-98\", \"k2\":\"k2-98\"},98\n" + + "99,{\"k1\":\"k1-99\", \"k2\":\"k2-99\"},99\n"; assertEquals(expected, resultSetToString(res)); res.close(); + } finally { executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); + htable.close(); + } + } + + @Test + public void testColumnKeyValueSelectQuery() throws Exception { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, col2_key text, col2_value text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("external_hbase_mapped_table"); + + HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE)) + .getConnection(testingCluster.getHBaseUtil().getConf()); + HTableInterface htable = hconn.getTable("external_hbase_table"); + + try { + for (int i = 0; i < 10; i++) { + Put put = new Put(Bytes.toBytes("rk-" + i)); + for (int j = 0; j < 5; j++) { + put.add("col2".getBytes(), ("key-" + j).getBytes(), Bytes.toBytes("value-" + j)); + } + put.add("col3".getBytes(), "".getBytes(), ("col3-value-" + i).getBytes()); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 >= 'rk-0'"); + assertResultSet(res); + cleanupQuery(res); } finally { + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); htable.close(); } } @@ -340,8 +382,8 @@ public void testRowFieldSelectQuery() throws Exception { ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 > 'field1-20'"); assertResultSet(res); cleanupQuery(res); - executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); htable.close(); } } @@ -469,8 +511,8 @@ public void testIndexPredication() throws Exception { ResultSet res = executeString("select * from hbase_mapped_table where rk >= '020' and rk <= '055'"); assertResultSet(res); res.close(); - executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE hbase_mapped_table PURGE").close(); htable.close(); hAdmin.close(); } @@ -512,8 +554,8 @@ public void testNonForwardQuery() throws Exception { ResultSet res = executeString("select * from hbase_mapped_table"); assertResultSet(res); res.close(); - executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE hbase_mapped_table PURGE").close(); hAdmin.close(); if (htable == null) { htable.close(); @@ -559,8 +601,8 @@ public void testJoin() throws Exception { "join default.lineitem b on a.col3 = b.l_orderkey"); assertResultSet(res); res.close(); - executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE hbase_mapped_table PURGE").close(); hAdmin.close(); if (htable != null) { htable.close(); @@ -601,8 +643,9 @@ public void testInsertInto() throws Exception { new byte[][]{null, Bytes.toBytes("a"), null, Bytes.toBytes("b")}, new boolean[]{false, false, false, true}, tableDesc.getSchema())); - executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + if (scanner != null) { scanner.close(); } @@ -661,9 +704,105 @@ public void testInsertIntoMultiRegion() throws Exception { new byte[][]{null, Bytes.toBytes("a")}, new boolean[]{false, false}, tableDesc.getSchema())); + } finally { executeString("DROP TABLE base_table PURGE").close(); executeString("DROP TABLE hbase_mapped_table PURGE").close(); + + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + + @Test + public void testInsertIntoColumnKeyValue() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col2_key text, col2_value text, col3 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("rk", Type.TEXT); + schema.addColumn("col2_key", Type.TEXT); + schema.addColumn("col2_value", Type.TEXT); + schema.addColumn("col3", Type.TEXT); + List datas = new ArrayList(); + for (int i = 20; i >= 0; i--) { + for (int j = 0; j < 3; j++) { + datas.add(i + "|ck-" + j + "|value-" + j + "|col3-" + i); + } + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("insert into hbase_mapped_table " + + "select rk, col2_key, col2_value, col3 from base_table ").close(); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col2")); + scan.addFamily(Bytes.toBytes("col3")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col2"), Bytes.toBytes("col3")}, + new byte[][]{null, null, null}, + new boolean[]{false, false, false}, tableDesc.getSchema())); + + ResultSet res = executeString("select * from hbase_mapped_table"); + + String expected = "rk,col2_key,col2_value,col3\n" + + "-------------------------------\n" + + "0,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-0\n" + + "1,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-1\n" + + "10,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-10\n" + + "11,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-11\n" + + "12,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-12\n" + + "13,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-13\n" + + "14,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-14\n" + + "15,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-15\n" + + "16,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-16\n" + + "17,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-17\n" + + "18,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-18\n" + + "19,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-19\n" + + "2,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-2\n" + + "20,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-20\n" + + "3,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-3\n" + + "4,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-4\n" + + "5,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-5\n" + + "6,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-6\n" + + "7,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-7\n" + + "8,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-8\n" + + "9,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-9\n"; + + assertEquals(expected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + if (scanner != null) { scanner.close(); } @@ -721,9 +860,10 @@ public void testInsertIntoMultiRegion2() throws Exception { new byte[][]{null, Bytes.toBytes("a")}, new boolean[]{false, false}, tableDesc.getSchema())); + } finally { executeString("DROP TABLE base_table PURGE").close(); executeString("DROP TABLE hbase_mapped_table PURGE").close(); - } finally { + if (scanner != null) { scanner.close(); } @@ -785,9 +925,10 @@ public void testInsertIntoMultiRegionMultiRowFields() throws Exception { new byte[][]{null, null, Bytes.toBytes("a")}, new boolean[]{false, false, false}, tableDesc.getSchema())); + } finally { executeString("DROP TABLE base_table PURGE").close(); executeString("DROP TABLE hbase_mapped_table PURGE").close(); - } finally { + if (scanner != null) { scanner.close(); } @@ -845,9 +986,10 @@ public void testInsertIntoBinaryMultiRegion() throws Exception { new byte[][]{null, Bytes.toBytes("a")}, new boolean[]{true, false}, tableDesc.getSchema())); + } finally { executeString("DROP TABLE base_table PURGE").close(); executeString("DROP TABLE hbase_mapped_table PURGE").close(); - } finally { + if (scanner != null) { scanner.close(); } @@ -934,8 +1076,9 @@ public void testInsertIntoRowField() throws Exception { new byte[][]{null, Bytes.toBytes("a"), Bytes.toBytes(""), Bytes.toBytes("b")}, new boolean[]{false, false, false, false}, tableDesc.getSchema())); - executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + if (scanner != null) { scanner.close(); } @@ -994,9 +1137,10 @@ public void testCATS() throws Exception { new byte[][]{null, Bytes.toBytes("a")}, new boolean[]{false, false}, tableDesc.getSchema())); + } finally { executeString("DROP TABLE base_table PURGE").close(); executeString("DROP TABLE hbase_mapped_table PURGE").close(); - } finally { + if (scanner != null) { scanner.close(); } @@ -1045,8 +1189,9 @@ public void testInsertIntoUsingPut() throws Exception { new byte[][]{null, Bytes.toBytes("a"), null, Bytes.toBytes("b")}, new boolean[]{false, false, false, true}, tableDesc.getSchema())); - executeString("DROP TABLE hbase_mapped_table PURGE").close(); } finally { + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + client.unsetSessionVariables(TUtil.newList(HBaseStorageManager.INSERT_PUT_MODE)); if (scanner != null) { @@ -1083,17 +1228,19 @@ private String resultSetToString(ResultScanner scanner, sb.append(", null"); } else { sb.append(", {"); + String delim = ""; for (Map.Entry valueEntry: values.entrySet()) { byte[] keyBytes = valueEntry.getKey(); byte[] valueBytes = valueEntry.getValue(); if (binaries[i]) { - sb.append("\"").append(keyBytes == null ? "" : Bytes.toLong(keyBytes)).append("\""); + sb.append(delim).append("\"").append(keyBytes == null ? "" : Bytes.toLong(keyBytes)).append("\""); sb.append(": \"").append(HBaseBinarySerializerDeserializer.deserialize(schema.getColumn(i), valueBytes)).append("\""); } else { - sb.append("\"").append(keyBytes == null ? "" : new String(keyBytes)).append("\""); + sb.append(delim).append("\"").append(keyBytes == null ? "" : new String(keyBytes)).append("\""); sb.append(": \"").append(HBaseTextSerializerDeserializer.deserialize(schema.getColumn(i), valueBytes)).append("\""); } + delim = ", "; } sb.append("}"); } diff --git a/tajo-core/src/test/resources/dataset/TestHBaseTable/splits.data b/tajo-core/src/test/resources/dataset/TestHBaseTable/splits.data new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result new file mode 100644 index 0000000000..82d4fd214e --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result @@ -0,0 +1,12 @@ +rk1,col2_key,col2_value,col3 +------------------------------- +rk-0,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-0 +rk-1,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-1 +rk-2,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-2 +rk-3,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-3 +rk-4,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-4 +rk-5,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-5 +rk-6,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-6 +rk-7,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-7 +rk-8,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-8 +rk-9,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-9 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result new file mode 100644 index 0000000000..a37593b9a5 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result @@ -0,0 +1,21 @@ +0, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-0"} +1, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-1"} +10, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-10"} +11, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-11"} +12, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-12"} +13, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-13"} +14, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-14"} +15, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-15"} +16, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-16"} +17, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-17"} +18, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-18"} +19, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-19"} +2, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-2"} +20, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-20"} +3, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-3"} +4, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-4"} +5, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-5"} +6, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-6"} +7, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-7"} +8, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-8"} +9, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-9"} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java index 3510e48b2c..f5e9b9ab27 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java @@ -38,7 +38,9 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public abstract class AbstractHBaseAppender implements Appender { protected Configuration conf; @@ -57,10 +59,20 @@ public abstract class AbstractHBaseAppender implements Appender { protected byte[][][] mappingColumnFamilies; protected boolean[] isBinaryColumns; protected boolean[] isRowKeyMappings; + protected boolean[] isColumnKeys; + protected boolean[] isColumnValues; protected int[] rowKeyFieldIndexes; protected int[] rowkeyColumnIndexes; protected char rowKeyDelimiter; + // the following four variables are used for ':key:' or ':value:' mapping + protected int[] columnKeyValueDataIndexes; + protected byte[][] columnKeyDatas; + protected byte[][] columnValueDatas; + protected byte[][] columnKeyCfNames; + + protected KeyValue[] keyValues; + public AbstractHBaseAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema, TableMeta meta, Path stagingDir) { this.conf = conf; @@ -93,10 +105,45 @@ public void init() throws IOException { rowkeyColumnIndexes = TUtil.toArray(rowkeyColumnIndexList); isBinaryColumns = columnMapping.getIsBinaryColumns(); + isColumnKeys = columnMapping.getIsColumnKeys(); + isColumnValues = columnMapping.getIsColumnValues(); rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); this.columnNum = schema.size(); + + // In the case of ':key:' or ':value:' KeyValue object should be set with the qualifier and value + // which are mapped to the same column family. + columnKeyValueDataIndexes = new int[isColumnKeys.length]; + int index = 0; + int numKeyValues = 0; + Map cfNameIndexMap = new HashMap(); + for (int i = 0; i < isColumnKeys.length; i++) { + if (isRowKeyMappings[i]) { + continue; + } + if (isColumnKeys[i] || isColumnValues[i]) { + String cfName = new String(mappingColumnFamilies[i][0]); + if (!cfNameIndexMap.containsKey(cfName)) { + cfNameIndexMap.put(cfName, index); + columnKeyValueDataIndexes[i] = index; + index++; + numKeyValues++; + } else { + columnKeyValueDataIndexes[i] = cfNameIndexMap.get(cfName); + } + } else { + numKeyValues++; + } + } + columnKeyCfNames = new byte[cfNameIndexMap.size()][]; + for (Map.Entry entry: cfNameIndexMap.entrySet()) { + columnKeyCfNames[entry.getValue()] = entry.getKey().getBytes(); + } + columnKeyDatas = new byte[cfNameIndexMap.size()][]; + columnValueDatas = new byte[cfNameIndexMap.size()][]; + + keyValues = new KeyValue[numKeyValues]; } private ByteArrayOutputStream bout = new ByteArrayOutputStream(); @@ -132,15 +179,33 @@ protected byte[] getRowKeyBytes(Tuple tuple) throws IOException { return rowkey; } - protected KeyValue getKeyValue(Tuple tuple, int index, byte[] rowkey) throws IOException { - Datum datum = tuple.get(index); - byte[] value; - if (isBinaryColumns[index]) { - value = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(index), datum); - } else { - value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(index), datum); + protected void readKeyValues(Tuple tuple, byte[] rowkey) throws IOException { + int keyValIndex = 0; + for (int i = 0; i < columnNum; i++) { + if (isRowKeyMappings[i]) { + continue; + } + Datum datum = tuple.get(i); + byte[] value; + if (isBinaryColumns[i]) { + value = HBaseBinarySerializerDeserializer.serialize(schema.getColumn(i), datum); + } else { + value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(i), datum); + } + + if (isColumnKeys[i]) { + columnKeyDatas[columnKeyValueDataIndexes[i]] = value; + } else if (isColumnValues[i]) { + columnValueDatas[columnKeyValueDataIndexes[i]] = value; + } else { + keyValues[keyValIndex] = new KeyValue(rowkey, mappingColumnFamilies[i][0], mappingColumnFamilies[i][1], value); + keyValIndex++; + } + } + + for (int i = 0; i < columnKeyDatas.length; i++) { + keyValues[keyValIndex++] = new KeyValue(rowkey, columnKeyCfNames[i], columnKeyDatas[i], columnValueDatas[i]); } - return new KeyValue(rowkey, mappingColumnFamilies[index][0], mappingColumnFamilies[index][1], value); } @Override diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java index ebd9af0c0c..2e8e1dd2ca 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -21,13 +21,16 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.util.Pair; +import org.apache.tajo.util.BytesUtils; import java.io.IOException; import java.util.ArrayList; import java.util.List; public class ColumnMapping { + public static final String KEY_COLUMN_MAPPING = "key"; + public static final String VALUE_COLUMN_MAPPING = "value"; + private TableMeta tableMeta; private Schema schema; private char rowKeyDelimiter; @@ -37,6 +40,8 @@ public class ColumnMapping { private int[] rowKeyFieldIndexes; private boolean[] isRowKeyMappings; private boolean[] isBinaryColumns; + private boolean[] isColumnKeys; + private boolean[] isColumnValues; // schema order -> 0: cf name, 1: column name -> name bytes private byte[][][] mappingColumns; @@ -57,65 +62,136 @@ public void init() throws IOException { isRowKeyMappings = new boolean[schema.size()]; rowKeyFieldIndexes = new int[schema.size()]; isBinaryColumns = new boolean[schema.size()]; + isColumnKeys = new boolean[schema.size()]; + isColumnValues = new boolean[schema.size()]; mappingColumns = new byte[schema.size()][][]; for (int i = 0; i < schema.size(); i++) { - isRowKeyMappings[i] = false; rowKeyFieldIndexes[i] = -1; - isBinaryColumns[i] = false; } - List> hbaseColumnMappings = parseColumnMapping(tableMeta); - if (hbaseColumnMappings == null || hbaseColumnMappings.isEmpty()) { + String columnMapping = tableMeta.getOption(HBaseStorageManager.META_COLUMNS_KEY, ""); + if (columnMapping == null || columnMapping.isEmpty()) { throw new IOException("'columns' property is required."); } - if (hbaseColumnMappings.size() != schema.getColumns().size()) { + String[] columnMappingTokens = columnMapping.split(","); + + if (columnMappingTokens.length != schema.getColumns().size()) { throw new IOException("The number of mapped HBase columns is great than the number of Tajo table columns"); } int index = 0; - for (Pair eachMapping: hbaseColumnMappings) { - String cfName = eachMapping.getFirst(); - String columnName = eachMapping.getSecond(); - + for (String eachToken: columnMappingTokens) { mappingColumns[index] = new byte[2][]; - RowKeyMapping rowKeyMapping = HBaseStorageManager.getRowKeyMapping(cfName, columnName); - if (rowKeyMapping != null) { - isRowKeyMappings[index] = true; - isBinaryColumns[index] = rowKeyMapping.isBinary(); - if (!cfName.isEmpty()) { - if (rowKeyDelimiter == 0) { - throw new IOException("hbase.rowkey.delimiter is required."); + byte[][] mappingTokens = BytesUtils.splitPreserveAllTokens(eachToken.trim().getBytes(), ':'); + + if (mappingTokens.length == 3) { + if (mappingTokens[0].length == 0) { + // cfname + throw new IOException("'column' attribute should be ':key:' or ':key:#b' " + + "or ':value:' or ':value:#b'"); + } + //:key: or :value: + if (mappingTokens[2].length != 0) { + String binaryOption = new String(mappingTokens[2]); + if ("#b".equals(binaryOption)) { + isBinaryColumns[index] = true; + } else { + throw new IOException("'column' attribute should be ':key:' or ':key:#b' " + + "or ':value:' or ':value:#b'"); } - rowKeyFieldIndexes[index] = Integer.parseInt(cfName); - } else { - rowKeyFieldIndexes[index] = -1; //rowkey is mapped a single column. } - } else { - isRowKeyMappings[index] = false; - - if (cfName != null) { - mappingColumns[index][0] = Bytes.toBytes(cfName); + mappingColumns[index][0] = mappingTokens[0]; + String keyOrValue = new String(mappingTokens[1]); + if (KEY_COLUMN_MAPPING.equalsIgnoreCase(keyOrValue)) { + isColumnKeys[index] = true; + } else if (VALUE_COLUMN_MAPPING.equalsIgnoreCase(keyOrValue)) { + isColumnValues[index] = true; + } else { + throw new IOException("'column' attribute should be ':key:' or ':value:'"); } - - if (columnName != null) { - String[] columnNameTokens = columnName.split("#"); - if (columnNameTokens[0].isEmpty()) { - mappingColumns[index][1] = null; + } else if (mappingTokens.length == 2) { + //: or : or :key + String cfName = new String(mappingTokens[0]); + String columnName = new String(mappingTokens[1]); + RowKeyMapping rowKeyMapping = getRowKeyMapping(cfName, columnName); + if (rowKeyMapping != null) { + isRowKeyMappings[index] = true; + isBinaryColumns[index] = rowKeyMapping.isBinary(); + if (!cfName.isEmpty()) { + if (rowKeyDelimiter == 0) { + throw new IOException("hbase.rowkey.delimiter is required."); + } + rowKeyFieldIndexes[index] = Integer.parseInt(cfName); } else { - mappingColumns[index][1] = Bytes.toBytes(columnNameTokens[0]); + rowKeyFieldIndexes[index] = -1; //rowkey is mapped a single column. } - if (columnNameTokens.length == 2 && "b".equals(columnNameTokens[1])) { - isBinaryColumns[index] = true; + } else { + if (cfName.isEmpty()) { + throw new IOException("'column' attribute should be ':key:' or ':value:'"); + } + if (cfName != null) { + mappingColumns[index][0] = Bytes.toBytes(cfName); + } + + if (columnName != null && !columnName.isEmpty()) { + String[] columnNameTokens = columnName.split("#"); + if (columnNameTokens[0].isEmpty()) { + mappingColumns[index][1] = null; + } else { + mappingColumns[index][1] = Bytes.toBytes(columnNameTokens[0]); + } + if (columnNameTokens.length == 2 && "b".equals(columnNameTokens[1])) { + isBinaryColumns[index] = true; + } } } + } else { + throw new IOException("'column' attribute '[cfname]:[qualfier]:"); } index++; + } // for loop + } + + public List getColumnFamilyNames() { + List cfNames = new ArrayList(); + + for (byte[][] eachCfName: mappingColumns) { + if (eachCfName != null && eachCfName.length > 0 && eachCfName[0] != null) { + String cfName = new String(eachCfName[0]); + if (!cfNames.contains(cfName)) { + cfNames.add(cfName); + } + } } + + return cfNames; + } + + private RowKeyMapping getRowKeyMapping(String cfName, String columnName) { + if (columnName == null || columnName.isEmpty()) { + return null; + } + + String[] tokens = columnName.split("#"); + if (!tokens[0].equalsIgnoreCase(KEY_COLUMN_MAPPING)) { + return null; + } + + RowKeyMapping rowKeyMapping = new RowKeyMapping(); + + if (tokens.length == 2 && "b".equals(tokens[1])) { + rowKeyMapping.setBinary(true); + } + + if (cfName != null && !cfName.isEmpty()) { + rowKeyMapping.setKeyFieldIndex(Integer.parseInt(cfName)); + } + return rowKeyMapping; } public char getRowKeyDelimiter() { @@ -142,37 +218,15 @@ public boolean[] getIsBinaryColumns() { return isBinaryColumns; } - /** - * Get column mapping data from tableMeta's option. - * First value of return is column family name and second value is column name which can be null. - * @param tableMeta - * @return - * @throws java.io.IOException - */ - public static List> parseColumnMapping(TableMeta tableMeta) throws IOException { - List> columnMappings = new ArrayList>(); - - String columnMapping = tableMeta.getOption(HBaseStorageManager.META_COLUMNS_KEY, ""); - if (columnMapping == null || columnMapping.trim().isEmpty()) { - return columnMappings; - } - - for (String eachToken: columnMapping.split(",")) { - String[] cfToken = eachToken.split(":"); - - String cfName = cfToken[0]; - String columnName = null; - if (cfToken.length == 2 && !cfToken[1].trim().isEmpty()) { - columnName = cfToken[1].trim(); - } - Pair mappingEntry = new Pair(cfName, columnName); - columnMappings.add(mappingEntry); - } + public String getHbaseTableName() { + return hbaseTableName; + } - return columnMappings; + public boolean[] getIsColumnKeys() { + return isColumnKeys; } - public String getHbaseTableName() { - return hbaseTableName; + public boolean[] getIsColumnValues() { + return isColumnValues; } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java deleted file mode 100644 index 82e91543d6..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseLazyTuple.java +++ /dev/null @@ -1,333 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.hbase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.client.Result; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.datum.ProtobufDatum; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.exception.UnsupportedException; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.util.BytesUtils; - -import java.util.Arrays; -import java.util.NavigableMap; - -public class HBaseLazyTuple implements Tuple, Cloneable { - private static final Log LOG = LogFactory.getLog(HBaseLazyTuple.class); - - private Datum[] values; - private Result result; - private byte[][][] mappingColumnFamilies; - private boolean[] isRowKeyMappings; - private boolean[] isBinaryColumns; - private int[] rowKeyFieldIndexes; - private char rowKeyDelimiter; - private Column[] schemaColumns; - - public HBaseLazyTuple(ColumnMapping columnMapping, - Column[] schemaColumns, - int[] targetIndexes, - Result result) { - values = new Datum[schemaColumns.length]; - mappingColumnFamilies = columnMapping.getMappingColumns(); - isRowKeyMappings = columnMapping.getIsRowKeyMappings(); - isBinaryColumns = columnMapping.getIsBinaryColumns(); - rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); - rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); - - this.result = result; - this.schemaColumns = schemaColumns; - } - - @Override - public int size() { - return values.length; - } - - @Override - public boolean contains(int fieldid) { - return false; - } - - @Override - public boolean isNull(int fieldid) { - return false; - } - - @Override - public void clear() { - values = new Datum[schemaColumns.length]; - } - - @Override - public void put(int fieldId, Datum value) { - values[fieldId] = value; - } - - @Override - public void put(int fieldId, Datum[] values) { - for (int i = fieldId, j = 0; j < values.length; i++, j++) { - this.values[i] = values[j]; - } - } - - @Override - public void put(int fieldId, Tuple tuple) { - for (int i = fieldId, j = 0; j < tuple.size(); i++, j++) { - values[i] = tuple.get(j); - } - } - - @Override - public void put(Datum[] values) { - System.arraycopy(values, 0, this.values, 0, size()); - } - - @Override - public Datum get(int fieldId) { - if (values[fieldId] != null) { - return values[fieldId]; - } - - byte[] value = null; - if (isRowKeyMappings[fieldId]) { - value = result.getRow(); - if (!isBinaryColumns[fieldId] && rowKeyFieldIndexes[fieldId] >= 0) { - int rowKeyFieldIndex = rowKeyFieldIndexes[fieldId]; - - byte[][] rowKeyFields = BytesUtils.splitPreserveAllTokens(value, rowKeyDelimiter); - - if (rowKeyFields.length < rowKeyFieldIndex) { - values[fieldId] = NullDatum.get(); - return values[fieldId]; - } else { - value = rowKeyFields[rowKeyFieldIndex]; - } - } - } else { - if (mappingColumnFamilies[fieldId][1] == null) { - NavigableMap cfMap = result.getFamilyMap(mappingColumnFamilies[fieldId][0]); - if (cfMap != null && !cfMap.isEmpty()) { - int count = 0; - String delim = ""; - - if (cfMap.size() == 0) { - values[fieldId] = NullDatum.get(); - return values[fieldId]; - } else if (cfMap.size() == 1) { - // If a column family is mapped without column name like "cf1:" and the number of cells is one, - // return value is flat format not json format. - NavigableMap.Entry entry = cfMap.entrySet().iterator().next(); - byte[] entryKey = entry.getKey(); - byte[] entryValue = entry.getValue(); - if (entryKey == null || entryKey.length == 0) { - try { - if (isBinaryColumns[fieldId]) { - values[fieldId] = HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue); - } else { - values[fieldId] = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue); - } - return values[fieldId]; - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - } - StringBuilder sb = new StringBuilder(); - sb.append("{"); - for (NavigableMap.Entry entry : cfMap.entrySet()) { - byte[] entryKey = entry.getKey(); - byte[] entryValue = entry.getValue(); - - String keyText = new String(entryKey); - String valueText = null; - if (entryValue != null) { - try { - if (isBinaryColumns[fieldId]) { - valueText = HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue).asChars(); - } else { - valueText = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue).asChars(); - } - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - sb.append(delim).append("\"").append(keyText).append("\":\"").append(valueText).append("\""); - delim = ", "; - if (count > 100) { - break; - } - } //end of for - sb.append("}"); - values[fieldId] = new TextDatum(sb.toString()); - return values[fieldId]; - } else { - value = null; - } - } else { - value = result.getValue(mappingColumnFamilies[fieldId][0], mappingColumnFamilies[fieldId][1]); - } - } - - if (value == null) { - values[fieldId] = NullDatum.get(); - } else { - try { - if (isBinaryColumns[fieldId]) { - values[fieldId] = HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], value); - } else { - values[fieldId] = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], value); - } - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - - return values[fieldId]; - } - - @Override - public void setOffset(long offset) { - } - - @Override - public long getOffset() { - return 0; - } - - @Override - public boolean getBool(int fieldId) { - return get(fieldId).asBool(); - } - - @Override - public byte getByte(int fieldId) { - return get(fieldId).asByte(); - } - - @Override - public char getChar(int fieldId) { - return get(fieldId).asChar(); - } - - @Override - public byte [] getBytes(int fieldId) { - return get(fieldId).asByteArray(); - } - - @Override - public short getInt2(int fieldId) { - return get(fieldId).asInt2(); - } - - @Override - public int getInt4(int fieldId) { - return get(fieldId).asInt4(); - } - - @Override - public long getInt8(int fieldId) { - return get(fieldId).asInt8(); - } - - @Override - public float getFloat4(int fieldId) { - return get(fieldId).asFloat4(); - } - - @Override - public double getFloat8(int fieldId) { - return get(fieldId).asFloat8(); - } - - @Override - public String getText(int fieldId) { - return get(fieldId).asChars(); - } - - @Override - public ProtobufDatum getProtobufDatum(int fieldId) { - throw new UnsupportedException(); - } - - @Override - public char[] getUnicodeChars(int fieldId) { - return get(fieldId).asUnicodeChars(); - } - - public String toString() { - boolean first = true; - StringBuilder str = new StringBuilder(); - str.append("("); - Datum d; - for (int i = 0; i < values.length; i++) { - d = get(i); - if (d != null) { - if (first) { - first = false; - } else { - str.append(", "); - } - str.append(i) - .append("=>") - .append(d); - } - } - str.append(")"); - return str.toString(); - } - - @Override - public int hashCode() { - return Arrays.hashCode(values); - } - - @Override - public Datum[] getValues() { - Datum[] datums = new Datum[values.length]; - for (int i = 0; i < values.length; i++) { - datums[i] = get(i); - } - return datums; - } - - @Override - public Tuple clone() throws CloneNotSupportedException { - HBaseLazyTuple lazyTuple = (HBaseLazyTuple) super.clone(); - lazyTuple.values = getValues(); //shallow copy - return lazyTuple; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof Tuple) { - Tuple other = (Tuple) obj; - return Arrays.equals(getValues(), other.getValues()); - } - return false; - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java index 3f73e5cba9..674e002212 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java @@ -37,6 +37,7 @@ public class HBasePutAppender extends AbstractHBaseAppender { private HTableInterface htable; + private long totalNumBytes; public HBasePutAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema, TableMeta meta, Path stagingDir) { @@ -55,12 +56,13 @@ public void init() throws IOException { htable.setWriteBufferSize(5 * 1024 * 1024); } - long totalNumBytes; @Override public void addTuple(Tuple tuple) throws IOException { byte[] rowkey = getRowKeyBytes(tuple); totalNumBytes += rowkey.length; Put put = new Put(rowkey); + readKeyValues(tuple, rowkey); + for (int i = 0; i < columnNum; i++) { if (isRowKeyMappings[i]) { continue; @@ -72,8 +74,20 @@ public void addTuple(Tuple tuple) throws IOException { } else { value = HBaseTextSerializerDeserializer.serialize(schema.getColumn(i), datum); } - put.add(mappingColumnFamilies[i][0], mappingColumnFamilies[i][1], value); - totalNumBytes += value.length; + + if (isColumnKeys[i]) { + columnKeyDatas[columnKeyValueDataIndexes[i]] = value; + } else if (isColumnValues[i]) { + columnValueDatas[columnKeyValueDataIndexes[i]] = value; + } else { + put.add(mappingColumnFamilies[i][0], mappingColumnFamilies[i][1], value); + totalNumBytes += value.length; + } + } + + for (int i = 0; i < columnKeyDatas.length; i++) { + put.add(columnKeyCfNames[i], columnKeyDatas[i], columnValueDatas[i]); + totalNumBytes += columnKeyDatas[i].length + columnValueDatas[i].length; } htable.put(put); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index b042520765..7077b697ba 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -31,18 +31,27 @@ import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.datum.TextDatum; import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Bytes; +import org.apache.tajo.util.BytesUtils; import java.io.IOException; +import java.util.Collection; +import java.util.NavigableMap; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; public class HBaseScanner implements Scanner { private static final Log LOG = LogFactory.getLog(HBaseScanner.class); - private static final int DEFAULT_FETCH_SZIE = 1000; + private static final int DEFAULT_FETCH_SIZE = 1000; + private static final int MAX_LIST_SIZE = 100; protected boolean inited = false; private TajoConf conf; @@ -67,6 +76,15 @@ public class HBaseScanner implements Scanner { private int numRows = 0; + private byte[][][] mappingColumnFamilies; + private boolean[] isRowKeyMappings; + private boolean[] isBinaryColumns; + private boolean[] isColumnKeys; + private boolean[] isColumnValues; + + private int[] rowKeyFieldIndexes; + private char rowKeyDelimiter; + public HBaseScanner (Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException { this.conf = (TajoConf)conf; this.schema = schema; @@ -90,7 +108,7 @@ public void init() throws IOException { } } - scanFetchSize = Integer.parseInt(meta.getOption("hbase.scanner.fetch,size", "" + DEFAULT_FETCH_SZIE)); + scanFetchSize = Integer.parseInt(meta.getOption("hbase.scanner.fetch,size", "" + DEFAULT_FETCH_SIZE)); if (targets == null) { targets = schema.toArray(); } @@ -102,6 +120,15 @@ public void init() throws IOException { targetIndexes[index++] = schema.getColumnId(eachTargetColumn.getQualifiedName()); } + mappingColumnFamilies = columnMapping.getMappingColumns(); + isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + isBinaryColumns = columnMapping.getIsBinaryColumns(); + isColumnKeys = columnMapping.getIsColumnKeys(); + isColumnValues = columnMapping.getIsColumnValues(); + + rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); + rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); + hbaseConf = HBaseStorageManager.getHBaseConfiguration(conf, meta); initScanner(); @@ -158,8 +185,170 @@ public Tuple next() throws IOException { } Result result = scanResults[scanResultIndex++]; + Tuple resultTuple = new VTuple(schema.size()); + for (int i = 0; i < targetIndexes.length; i++) { + resultTuple.put(targetIndexes[i], getDatum(result, targetIndexes[i])); + } numRows++; - return new HBaseLazyTuple(columnMapping, schemaColumns, targetIndexes, result); + return resultTuple; + } + + private Datum getDatum(Result result, int fieldId) throws IOException { + byte[] value = null; + if (isRowKeyMappings[fieldId]) { + value = result.getRow(); + if (!isBinaryColumns[fieldId] && rowKeyFieldIndexes[fieldId] >= 0) { + int rowKeyFieldIndex = rowKeyFieldIndexes[fieldId]; + + byte[][] rowKeyFields = BytesUtils.splitPreserveAllTokens(value, rowKeyDelimiter); + + if (rowKeyFields.length < rowKeyFieldIndex) { + return NullDatum.get(); + } else { + value = rowKeyFields[rowKeyFieldIndex]; + } + } + } else { + if (isColumnKeys[fieldId]) { + NavigableMap cfMap = result.getFamilyMap(mappingColumnFamilies[fieldId][0]); + if (cfMap != null) { + Set keySet = cfMap.keySet(); + if (keySet.size() == 1) { + try { + return HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], keySet.iterator().next()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } else { + StringBuilder sb = new StringBuilder(); + sb.append("["); + int count = 0; + for (byte[] eachKey : keySet) { + if (count > 0) { + sb.append(", "); + } + Datum datum = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], eachKey); + sb.append("\"").append(datum.asChars()).append("\""); + count++; + if (count > MAX_LIST_SIZE) { + break; + } + } + sb.append("]"); + return new TextDatum(sb.toString()); + } + } + } else if (isColumnValues[fieldId]) { + NavigableMap cfMap = result.getFamilyMap(mappingColumnFamilies[fieldId][0]); + if (cfMap != null) { + Collection valueList = cfMap.values(); + if (valueList.size() == 1) { + try { + return HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], valueList.iterator().next()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } else { + StringBuilder sb = new StringBuilder(); + sb.append("["); + int count = 0; + for (byte[] eachValue : valueList) { + if (count > 0) { + sb.append(", "); + } + Datum datum = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], eachValue); + sb.append("\"").append(datum.asChars()).append("\""); + count++; + if (count > MAX_LIST_SIZE) { + break; + } + } + sb.append("]"); + return new TextDatum(sb.toString()); + } + } + } else { + if (mappingColumnFamilies[fieldId][1] == null) { + NavigableMap cfMap = result.getFamilyMap(mappingColumnFamilies[fieldId][0]); + if (cfMap != null && !cfMap.isEmpty()) { + int count = 0; + String delim = ""; + + if (cfMap.size() == 0) { + return NullDatum.get(); + } else if (cfMap.size() == 1) { + // If a column family is mapped without column name like "cf1:" and the number of cells is one, + // return value is flat format not json format. + NavigableMap.Entry entry = cfMap.entrySet().iterator().next(); + byte[] entryKey = entry.getKey(); + byte[] entryValue = entry.getValue(); + if (entryKey == null || entryKey.length == 0) { + try { + if (isBinaryColumns[fieldId]) { + return HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue); + } else { + return HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + } + StringBuilder sb = new StringBuilder(); + sb.append("{"); + for (NavigableMap.Entry entry : cfMap.entrySet()) { + byte[] entryKey = entry.getKey(); + byte[] entryValue = entry.getValue(); + + String keyText = new String(entryKey); + String valueText = null; + if (entryValue != null) { + try { + if (isBinaryColumns[fieldId]) { + valueText = HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue).asChars(); + } else { + valueText = HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], entryValue).asChars(); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + sb.append(delim).append("\"").append(keyText).append("\":\"").append(valueText).append("\""); + delim = ", "; + count++; + if (count > MAX_LIST_SIZE) { + break; + } + } //end of for + sb.append("}"); + return new TextDatum(sb.toString()); + } else { + value = null; + } + } else { + value = result.getValue(mappingColumnFamilies[fieldId][0], mappingColumnFamilies[fieldId][1]); + } + } + } + + if (value == null) { + return NullDatum.get(); + } else { + try { + if (isBinaryColumns[fieldId]) { + return HBaseBinarySerializerDeserializer.deserialize(schemaColumns[fieldId], value); + } else { + return HBaseTextSerializerDeserializer.deserialize(schemaColumns[fieldId], value); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } } @Override diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index d9ee5c3a66..39b6b77812 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -60,7 +60,6 @@ public class HBaseStorageManager extends StorageManager { public static final String META_SPLIT_ROW_KEYS_KEY = "hbase.split.rowkeys"; public static final String META_SPLIT_ROW_KEYS_FILE_KEY = "hbase.split.rowkeys.file"; public static final String META_ZK_QUORUM_KEY = "hbase.zookeeper.quorum"; - public static final String ROWKEY_COLUMN_MAPPING = "key"; public static final String META_ROWKEY_DELIMITER = "hbase.rowkey.delimiter"; public static final String INSERT_PUT_MODE = "tajo.hbase.insert.put.mode"; @@ -119,12 +118,21 @@ private void createTable(TableMeta tableMeta, Schema schema, } if (numRowKeys > 1) { for (int i = 0; i < isRowKeyMappings.length; i++) { - if (schema.getColumn(i).getDataType().getType() != Type.TEXT) { + if (isRowKeyMappings[i] && schema.getColumn(i).getDataType().getType() != Type.TEXT) { throw new IOException("Key field type should be TEXT type."); } } } + for (int i = 0; i < isRowKeyMappings.length; i++) { + if (columnMapping.getIsColumnKeys()[i] && schema.getColumn(i).getDataType().getType() != Type.TEXT) { + throw new IOException("Column key field(':key:') type should be TEXT type."); + } + if (columnMapping.getIsColumnValues()[i] && schema.getColumn(i).getDataType().getType() != Type.TEXT) { + throw new IOException("Column value field((':value:') type should be TEXT type."); + } + } + Configuration hConf = getHBaseConfiguration(conf, tableMeta); HBaseAdmin hAdmin = new HBaseAdmin(hConf); @@ -144,7 +152,7 @@ private void createTable(TableMeta tableMeta, Schema schema, tableColumnFamilies.add(eachColumn.getNameAsString()); } - Collection mappingColumnFamilies = getColumnFamilies(mappedColumns); + Collection mappingColumnFamilies =columnMapping.getColumnFamilyNames(); if (mappingColumnFamilies.isEmpty()) { throw new IOException("HBase mapped table is required a '" + META_COLUMNS_KEY + "' attribute."); } @@ -266,29 +274,6 @@ private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) thro return null; } - private static List getColumnFamilies(String columnMapping) { - // columnMapping can have a duplicated column name as CF1:a, CF1:b - List columnFamilies = new ArrayList(); - - if (columnMapping == null) { - return columnFamilies; - } - - for (String eachToken: columnMapping.split(",")) { - String[] cfTokens = eachToken.trim().split(":"); - if (cfTokens.length == 2 && cfTokens[1] != null && getRowKeyMapping(cfTokens[0], cfTokens[1].trim()) != null) { - // rowkey - continue; - } - if (!columnFamilies.contains(cfTokens[0])) { - String[] binaryTokens = cfTokens[0].split("#"); - columnFamilies.add(binaryTokens[0]); - } - } - - return columnFamilies; - } - public static Configuration getHBaseConfiguration(Configuration conf, TableMeta tableMeta) throws IOException { String zkQuorum = tableMeta.getOption(META_ZK_QUORUM_KEY, ""); if (zkQuorum == null || zkQuorum.trim().isEmpty()) { @@ -314,13 +299,11 @@ public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema } TableName hTableName = TableName.valueOf(hbaseTableName); - String columnMapping = tableMeta.getOption(META_COLUMNS_KEY, ""); - if (columnMapping != null && columnMapping.split(",").length > schema.size()) { - throw new IOException("Columns property has more entry than Tajo table columns"); - } + ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta); + HTableDescriptor hTableDescriptor = new HTableDescriptor(hTableName); - Collection columnFamilies = getColumnFamilies(columnMapping); + Collection columnFamilies = columnMapping.getColumnFamilyNames(); //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. if (columnFamilies.isEmpty()) { for (Column eachColumn: schema.getColumns()) { @@ -502,28 +485,6 @@ public List getSplits(String fragmentId, TableDesc tableDesc, ScanNode } } - public static RowKeyMapping getRowKeyMapping(String cfName, String columnName) { - if (columnName == null || columnName.isEmpty()) { - return null; - } - - String[] tokens = columnName.split("#"); - if (!tokens[0].equalsIgnoreCase(ROWKEY_COLUMN_MAPPING)) { - return null; - } - - RowKeyMapping rowKeyMapping = new RowKeyMapping(); - - if (tokens.length == 2 && "b".equals(tokens[1])) { - rowKeyMapping.setBinary(true); - } - - if (cfName != null && !cfName.isEmpty()) { - rowKeyMapping.setKeyFieldIndex(Integer.parseInt(cfName)); - } - return rowKeyMapping; - } - private byte[] serialize(ColumnMapping columnMapping, IndexPredication indexPredication, Datum datum) throws IOException { if (columnMapping.getIsBinaryColumns()[indexPredication.getColumnId()]) { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java index fb88f6cb00..29f434c6a4 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java @@ -94,6 +94,7 @@ public void init() throws IOException { boolean first = true; TreeSet kvSet = new TreeSet(KeyValue.COMPARATOR); + @Override public void addTuple(Tuple tuple) throws IOException { Datum datum; @@ -119,11 +120,12 @@ public void addTuple(Tuple tuple) throws IOException { first = false; keyWritable.set(rowkey); - for (int i = 0; i < columnNum; i++) { - if (isRowKeyMappings[i]) { - continue; + + readKeyValues(tuple, rowkey); + if (keyValues != null) { + for (KeyValue eachKeyVal: keyValues) { + kvSet.add(eachKeyVal); } - kvSet.add(getKeyValue(tuple, i, rowkey)); } } diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java new file mode 100644 index 0000000000..77967743a3 --- /dev/null +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.hbase; + +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.util.KeyValueSet; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestColumnMapping { + @Test + public void testColumnKeyValueMapping() throws Exception { + KeyValueSet keyValueSet = new KeyValueSet(); + keyValueSet.set(HBaseStorageManager.META_TABLE_KEY, "test"); + keyValueSet.set(HBaseStorageManager.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:"); + + Schema schema = new Schema(); + schema.addColumn("c1", Type.TEXT); + schema.addColumn("c2", Type.TEXT); + schema.addColumn("c3", Type.TEXT); + schema.addColumn("c4", Type.TEXT); + + TableMeta tableMeta = new TableMeta(StoreType.HBASE, keyValueSet); + + ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta); + + List cfNames = columnMapping.getColumnFamilyNames(); + assertEquals(2, cfNames.size()); + assertEquals("col2", cfNames.get(0)); + assertEquals("col3", cfNames.get(1)); + + for (int i = 0; i < columnMapping.getIsBinaryColumns().length; i++) { + if (i == 2) { + assertTrue(columnMapping.getIsBinaryColumns()[i]); + } else { + assertFalse(columnMapping.getIsBinaryColumns()[i]); + } + } + + for (int i = 0; i < columnMapping.getIsRowKeyMappings().length; i++) { + if (i == 0) { + assertTrue(columnMapping.getIsRowKeyMappings()[i]); + } else { + assertFalse(columnMapping.getIsRowKeyMappings()[i]); + } + } + + String[] expectedColumnNames = { null, null, null, null}; + for (int i = 0; i < schema.size(); i++) { + String columnName = columnMapping.getMappingColumns()[i][1] == null ? null : + new String(columnMapping.getMappingColumns()[i][1]); + assertEquals(expectedColumnNames[i], columnName); + } + + for (int i = 0; i < schema.size(); i++) { + if (i == 1) { + assertTrue(columnMapping.getIsColumnKeys()[i]); + } else { + assertFalse(columnMapping.getIsColumnKeys()[i]); + } + } + + for (int i = 0; i < schema.size(); i++) { + if (i == 2) { + assertTrue(columnMapping.getIsColumnValues()[i]); + } else { + assertFalse(columnMapping.getIsColumnValues()[i]); + } + } + } +} From bfe44fd7953cdeee9c51912aa9f20331cb0c002f Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 20:54:53 +0900 Subject: [PATCH 16/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Fix queryunit.jsp error --- .../engine/function/string/toCharInt.java | 55 +++++++++++++++++++ .../resources/webapps/worker/queryunit.jsp | 5 +- 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java new file mode 100644 index 0000000000..fa6a502999 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.string; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.storage.Tuple; + +import java.text.DecimalFormat; + +@Description( + functionName = "to_char", + description = "convert integer to string.", + example = "> SELECT to_char(125, '00999');\n" + + "00125", + returnType = TajoDataTypes.Type.TEXT, + paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.INT4, TajoDataTypes.Type.TEXT})} +) + +public class ToCharInt extends GeneralFunction { + DecimalFormat df = null; + + public ToCharInt() { + super(new Column[]{new Column("val", TajoDataTypes.Type.INT4), new Column("format", TajoDataTypes.Type.TEXT)}); + } + + @Override + public Datum eval(Tuple params) { + if (df != null) { + df = new DecimalFormat(params.get(0).asChars()); + } + return new TextDatum(df.format(params.get(1).asInt4())); + } +} diff --git a/tajo-core/src/main/resources/webapps/worker/queryunit.jsp b/tajo-core/src/main/resources/webapps/worker/queryunit.jsp index 18a67d80d4..49635d117e 100644 --- a/tajo-core/src/main/resources/webapps/worker/queryunit.jsp +++ b/tajo-core/src/main/resources/webapps/worker/queryunit.jsp @@ -41,6 +41,7 @@ <%@ page import="java.text.SimpleDateFormat" %> <%@ page import="java.util.Map" %> <%@ page import="java.util.Set" %> +<%@ page import="org.apache.tajo.storage.fragment.Fragment" %> <% String paramQueryId = request.getParameter("queryId"); @@ -102,8 +103,8 @@ String fragmentInfo = ""; String delim = ""; for (CatalogProtos.FragmentProto eachFragment : queryUnit.getAllFragments()) { - FileFragment fileFragment = FragmentConvertor.convert(FileFragment.class, eachFragment); - fragmentInfo += delim + fileFragment.toString(); + Fragment fragment = FragmentConvertor.convert(tajoWorker.getConfig(), eachFragment); + fragmentInfo += delim + fragment.toString(); delim = "
"; } From ce9de5231a09ad7013f7165258395c68adee8bc4 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 20:58:41 +0900 Subject: [PATCH 17/29] remove ToCharInt function --- .../engine/function/string/toCharInt.java | 55 ------------------- 1 file changed, 55 deletions(-) delete mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java deleted file mode 100644 index fa6a502999..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/toCharInt.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.function.string; - -import org.apache.tajo.catalog.Column; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.annotation.Description; -import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.plan.function.GeneralFunction; -import org.apache.tajo.storage.Tuple; - -import java.text.DecimalFormat; - -@Description( - functionName = "to_char", - description = "convert integer to string.", - example = "> SELECT to_char(125, '00999');\n" - + "00125", - returnType = TajoDataTypes.Type.TEXT, - paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.INT4, TajoDataTypes.Type.TEXT})} -) - -public class ToCharInt extends GeneralFunction { - DecimalFormat df = null; - - public ToCharInt() { - super(new Column[]{new Column("val", TajoDataTypes.Type.INT4), new Column("format", TajoDataTypes.Type.TEXT)}); - } - - @Override - public Datum eval(Tuple params) { - if (df != null) { - df = new DecimalFormat(params.get(0).asChars()); - } - return new TextDatum(df.format(params.get(1).asInt4())); - } -} From 73126be997dce5ef77043de4a0aad99eb72c1031 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 21:53:52 +0900 Subject: [PATCH 18/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Fix index predication bug --- .../engine/function/string/ToCharLong.java | 55 +++++ .../TestStringOperatorsAndFunctions.java | 9 + .../tajo/engine/query/TestHBaseTable.java | 228 +++++++++++------- .../tajo/storage/hbase/ColumnMapping.java | 7 + .../storage/hbase/HBaseStorageManager.java | 8 +- 5 files changed, 215 insertions(+), 92 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToCharLong.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToCharLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToCharLong.java new file mode 100644 index 0000000000..5fed940c47 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToCharLong.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.string; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.storage.Tuple; + +import java.text.DecimalFormat; + +@Description( + functionName = "to_char", + description = "convert integer to string.", + example = "> SELECT to_char(125, '00999');\n" + + "00125", + returnType = TajoDataTypes.Type.TEXT, + paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.INT8, TajoDataTypes.Type.TEXT})} +) + +public class ToCharLong extends GeneralFunction { + DecimalFormat df = null; + + public ToCharLong() { + super(new Column[]{new Column("val", TajoDataTypes.Type.INT8), new Column("format", TajoDataTypes.Type.TEXT)}); + } + + @Override + public Datum eval(Tuple params) { + if (df == null) { + df = new DecimalFormat(params.get(1).asChars()); + } + return new TextDatum(df.format(params.get(0).asInt8())); + } +} \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java index 7f402a13bc..4e13f9fdca 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java @@ -25,6 +25,7 @@ import org.junit.Test; import java.io.IOException; +import java.text.DecimalFormat; import static org.apache.tajo.common.TajoDataTypes.Type.*; @@ -613,4 +614,12 @@ public void testConcat_ws() throws IOException { testSimpleEval("select concat_ws(',', '22', '33', '33') ", new String[]{"22,33,33"}); testSimpleEval("select concat_ws(',', null, '22') ", new String[]{"22"}); } + + @Test + public void testToChar() throws IOException { + DecimalFormat df = new DecimalFormat("0000000000"); + System.out.println(">>>>>>>>>>>" + df.format(1234)); + testSimpleEval("select to_char('123456'::INT8, '0000000000') ", new String[]{"0000123456"}); + + } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index e2d8e28e00..bb5dc68497 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -402,7 +402,7 @@ public void testIndexPredication() throws Exception { assertTableExists("hbase_mapped_table"); - HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); hAdmin.tableExists("hbase_table"); HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); @@ -420,93 +420,7 @@ public void testIndexPredication() throws Exception { put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); htable.put(put); } - - TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); - - // where rk >= '020' and rk <= '055' - ScanNode scanNode = new ScanNode(1); - EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("020"))); - EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("055"))); - EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); - - scanNode.setQual(evalNodeA); - - StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE); - List fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); - - assertEquals(2, fragments.size()); - HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); - assertEquals("020", new String(fragment1.getStartRow())); - assertEquals("040", new String(fragment1.getStopRow())); - - HBaseFragment fragment2 = (HBaseFragment) fragments.get(1); - assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("055", new String(fragment2.getStopRow())); - - - // where (rk >= '020' and rk <= '055') or rk = '075' - EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("075"))); - EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); - scanNode.setQual(evalNodeB); - fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); - assertEquals(3, fragments.size()); - fragment1 = (HBaseFragment) fragments.get(0); - assertEquals("020", new String(fragment1.getStartRow())); - assertEquals("040", new String(fragment1.getStopRow())); - - fragment2 = (HBaseFragment) fragments.get(1); - assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("055", new String(fragment2.getStopRow())); - - HBaseFragment fragment3 = (HBaseFragment) fragments.get(2); - assertEquals("075", new String(fragment3.getStartRow())); - assertEquals("075", new String(fragment3.getStopRow())); - - - // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078') - EvalNode evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("072"))); - EvalNode evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("078"))); - EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); - EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); - scanNode.setQual(evalNodeD); - fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); - assertEquals(3, fragments.size()); - - fragment1 = (HBaseFragment) fragments.get(0); - assertEquals("020", new String(fragment1.getStartRow())); - assertEquals("040", new String(fragment1.getStopRow())); - - fragment2 = (HBaseFragment) fragments.get(1); - assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("055", new String(fragment2.getStopRow())); - - fragment3 = (HBaseFragment) fragments.get(2); - assertEquals("072", new String(fragment3.getStartRow())); - assertEquals("078", new String(fragment3.getStopRow())); - - // where (rk >= '020' and rk <= '055') or (rk >= '057' and rk <= '059') - evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("057"))); - evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), - new ConstEval(new TextDatum("059"))); - evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); - evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); - scanNode.setQual(evalNodeD); - fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); - assertEquals(2, fragments.size()); - - fragment1 = (HBaseFragment) fragments.get(0); - assertEquals("020", new String(fragment1.getStartRow())); - assertEquals("040", new String(fragment1.getStopRow())); - - fragment2 = (HBaseFragment) fragments.get(1); - assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("059", new String(fragment2.getStopRow())); + assertIndexPredication(); ResultSet res = executeString("select * from hbase_mapped_table where rk >= '020' and rk <= '055'"); assertResultSet(res); @@ -518,6 +432,144 @@ public void testIndexPredication() throws Exception { } } + @Test + public void testCompositeRowIndexPredication() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, rk2 text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a,col2:,col3:b', " + + "'hbase.split.rowkeys'='010,040,060,080', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + + assertTableExists("hbase_mapped_table"); + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); + hAdmin.tableExists("hbase_table"); + + HTable htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + try { + org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); + assertEquals(5, keys.getFirst().length); + + DecimalFormat df = new DecimalFormat("000"); + for (int i = 0; i < 100; i++) { + Put put = new Put((df.format(i) + "_" + df.format(i)).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } + assertIndexPredication(); + } finally { + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + htable.close(); + hAdmin.close(); + } + } + + private void assertIndexPredication() throws Exception { + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + ScanNode scanNode = new ScanNode(1); + + // where rk = '021' + EvalNode evalNodeEq = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("021"))); + scanNode.setQual(evalNodeEq); + StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE); + List fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + assertEquals(1, fragments.size()); + assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow())); + assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStopRow())); + + // where rk >= '020' and rk <= '055' + EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("020"))); + EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("055"))); + EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); + scanNode.setQual(evalNodeA); + + fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + assertEquals(2, fragments.size()); + HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + HBaseFragment fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("055", new String(fragment2.getStopRow())); + + + // where (rk >= '020' and rk <= '055') or rk = '075' + EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("075"))); + EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); + scanNode.setQual(evalNodeB); + fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + assertEquals(3, fragments.size()); + fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("055", new String(fragment2.getStopRow())); + + HBaseFragment fragment3 = (HBaseFragment) fragments.get(2); + assertEquals("075", new String(fragment3.getStartRow())); + assertEquals("075", new String(fragment3.getStopRow())); + + + // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078') + EvalNode evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("072"))); + EvalNode evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("078"))); + EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); + EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); + scanNode.setQual(evalNodeD); + fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + assertEquals(3, fragments.size()); + + fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("055", new String(fragment2.getStopRow())); + + fragment3 = (HBaseFragment) fragments.get(2); + assertEquals("072", new String(fragment3.getStartRow())); + assertEquals("078", new String(fragment3.getStopRow())); + + // where (rk >= '020' and rk <= '055') or (rk >= '057' and rk <= '059') + evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("057"))); + evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), + new ConstEval(new TextDatum("059"))); + evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); + evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); + scanNode.setQual(evalNodeD); + fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + assertEquals(2, fragments.size()); + + fragment1 = (HBaseFragment) fragments.get(0); + assertEquals("020", new String(fragment1.getStartRow())); + assertEquals("040", new String(fragment1.getStopRow())); + + fragment2 = (HBaseFragment) fragments.get(1); + assertEquals("040", new String(fragment2.getStartRow())); + assertEquals("059", new String(fragment2.getStopRow())); + } + @Test public void testNonForwardQuery() throws Exception { String hostName = InetAddress.getLocalHost().getHostName(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java index 2e8e1dd2ca..b48bb22c53 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -46,6 +46,8 @@ public class ColumnMapping { // schema order -> 0: cf name, 1: column name -> name bytes private byte[][][] mappingColumns; + private int numRowKeys; + public ColumnMapping(Schema schema, TableMeta tableMeta) throws IOException { this.schema = schema; this.tableMeta = tableMeta; @@ -120,6 +122,7 @@ public void init() throws IOException { RowKeyMapping rowKeyMapping = getRowKeyMapping(cfName, columnName); if (rowKeyMapping != null) { isRowKeyMappings[index] = true; + numRowKeys++; isBinaryColumns[index] = rowKeyMapping.isBinary(); if (!cfName.isEmpty()) { if (rowKeyDelimiter == 0) { @@ -226,6 +229,10 @@ public boolean[] getIsColumnKeys() { return isColumnKeys; } + public int getNumRowKeys() { + return numRowKeys; + } + public boolean[] getIsColumnValues() { return isColumnValues; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 39b6b77812..86e2a64eea 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -334,15 +334,15 @@ public void purgeTable(TableDesc tableDesc) throws IOException { private Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + int[] rowKeyIndexes = columnMapping.getRowKeyFieldIndexes(); Column indexColumn = null; for (int i = 0; i < isRowKeyMappings.length; i++) { if (isRowKeyMappings[i]) { - if (indexColumn != null) { - //Currently only supports single rowkey. - return null; + if (columnMapping.getNumRowKeys() == 1 || + rowKeyIndexes[i] == 0) { + indexColumn = tableDesc.getSchema().getColumn(i); } - indexColumn = tableDesc.getSchema().getColumn(i); } } return new Column[]{indexColumn}; From f28586cfa88eaffab298b0eba38f7c3288dab098 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 3 Nov 2014 22:44:38 +0900 Subject: [PATCH 19/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Fix composite rowkey scanner --- .../tajo/engine/query/TestHBaseTable.java | 53 +++++++++++++++---- .../storage/hbase/HBaseStorageManager.java | 17 ++++-- .../hbase/TestHBaseStorageManager.java | 14 ++--- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index bb5dc68497..1a80fb1b6c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -22,6 +22,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoTestingCluster; @@ -420,11 +422,19 @@ public void testIndexPredication() throws Exception { put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); htable.put(put); } - assertIndexPredication(); + assertIndexPredication(false); ResultSet res = executeString("select * from hbase_mapped_table where rk >= '020' and rk <= '055'"); assertResultSet(res); res.close(); + + res = executeString("select * from hbase_mapped_table where rk = '021'"); + String expected = "rk,col1,col2,col3\n" + + "-------------------------------\n" + + "021,a-21,{\"k1\":\"k1-21\", \"k2\":\"k2-21\"},b-21\n"; + + assertEquals(expected, resultSetToString(res)); + res.close(); } finally { executeString("DROP TABLE hbase_mapped_table PURGE").close(); htable.close(); @@ -465,7 +475,28 @@ public void testCompositeRowIndexPredication() throws Exception { put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); htable.put(put); } - assertIndexPredication(); + + Scan scan = new Scan(); + scan.setStartRow("021".getBytes()); + scan.setStopRow(("021_" + new String(new char[]{Character.MAX_VALUE})).getBytes()); + Filter filter = new InclusiveStopFilter(scan.getStopRow()); + scan.setFilter(filter); + + ResultScanner scanner = htable.getScanner(scan); + Result result = scanner.next(); + assertNotNull(result); + assertEquals("021_021", new String(result.getRow())); + scanner.close(); + + assertIndexPredication(true); + + ResultSet res = executeString("select * from hbase_mapped_table where rk = '021'"); + String expected = "rk,rk2,col1,col2,col3\n" + + "-------------------------------\n" + + "021,021,a-21,{\"k1\":\"k1-21\", \"k2\":\"k2-21\"},b-21\n"; + + assertEquals(expected, resultSetToString(res)); + res.close(); } finally { executeString("DROP TABLE hbase_mapped_table PURGE").close(); htable.close(); @@ -473,7 +504,8 @@ public void testCompositeRowIndexPredication() throws Exception { } } - private void assertIndexPredication() throws Exception { + private void assertIndexPredication(boolean isCompositeRowKey) throws Exception { + String postFix = isCompositeRowKey ? "_" + new String(new char[]{Character.MAX_VALUE}) : ""; TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); ScanNode scanNode = new ScanNode(1); @@ -486,7 +518,7 @@ private void assertIndexPredication() throws Exception { List fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(1, fragments.size()); assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow())); - assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStopRow())); + assertEquals("021" + postFix, new String(((HBaseFragment)fragments.get(0)).getStopRow())); // where rk >= '020' and rk <= '055' EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), @@ -504,8 +536,7 @@ private void assertIndexPredication() throws Exception { HBaseFragment fragment2 = (HBaseFragment) fragments.get(1); assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("055", new String(fragment2.getStopRow())); - + assertEquals("055" + postFix, new String(fragment2.getStopRow())); // where (rk >= '020' and rk <= '055') or rk = '075' EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), @@ -520,11 +551,11 @@ private void assertIndexPredication() throws Exception { fragment2 = (HBaseFragment) fragments.get(1); assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("055", new String(fragment2.getStopRow())); + assertEquals("055" + postFix, new String(fragment2.getStopRow())); HBaseFragment fragment3 = (HBaseFragment) fragments.get(2); assertEquals("075", new String(fragment3.getStartRow())); - assertEquals("075", new String(fragment3.getStopRow())); + assertEquals("075" + postFix, new String(fragment3.getStopRow())); // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078') @@ -544,11 +575,11 @@ private void assertIndexPredication() throws Exception { fragment2 = (HBaseFragment) fragments.get(1); assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("055", new String(fragment2.getStopRow())); + assertEquals("055" + postFix, new String(fragment2.getStopRow())); fragment3 = (HBaseFragment) fragments.get(2); assertEquals("072", new String(fragment3.getStartRow())); - assertEquals("078", new String(fragment3.getStopRow())); + assertEquals("078" + postFix, new String(fragment3.getStopRow())); // where (rk >= '020' and rk <= '055') or (rk >= '057' and rk <= '059') evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), @@ -567,7 +598,7 @@ private void assertIndexPredication() throws Exception { fragment2 = (HBaseFragment) fragments.get(1); assertEquals("040", new String(fragment2.getStartRow())); - assertEquals("059", new String(fragment2.getStopRow())); + assertEquals("059" + postFix, new String(fragment2.getStopRow())); } @Test diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 86e2a64eea..1f4cc5a417 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -350,7 +350,9 @@ private Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { @Override public List getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException { - List indexPredications = getIndexPredications(tableDesc, scanNode); + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); + + List indexPredications = getIndexPredications(columnMapping, tableDesc, scanNode); Configuration hconf = getHBaseConfiguration(conf, tableDesc.getMeta()); HTable htable = null; HBaseAdmin hAdmin = null; @@ -371,7 +373,6 @@ public List getSplits(String fragmentId, TableDesc tableDesc, ScanNode return fragments; } - ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); List startRows; List stopRows; @@ -698,14 +699,15 @@ public String toString() { } } - public List getIndexPredications(TableDesc tableDesc, ScanNode scanNode) throws IOException { + public List getIndexPredications(ColumnMapping columnMapping, + TableDesc tableDesc, ScanNode scanNode) throws IOException { List indexPredications = new ArrayList(); Column[] indexableColumns = getIndexableColumns(tableDesc); if (indexableColumns != null && indexableColumns.length == 1) { // Currently supports only single index column. List> indexablePredicateList = findIndexablePredicateSet(scanNode, indexableColumns); for (Set eachEvalSet: indexablePredicateList) { - Pair indexPredicationValues = getIndexablePredicateValue(eachEvalSet); + Pair indexPredicationValues = getIndexablePredicateValue(columnMapping, eachEvalSet); if (indexPredicationValues != null) { IndexPredication indexPredication = new IndexPredication(); indexPredication.setColumn(indexableColumns[0]); @@ -800,7 +802,8 @@ public static boolean isIndexableOperator(EvalNode expr) { expr.getType() == EvalType.BETWEEN; } - public Pair getIndexablePredicateValue(Set evalNodes) { + public Pair getIndexablePredicateValue(ColumnMapping columnMapping, + Set evalNodes) { Datum startDatum = null; Datum endDatum = null; for (EvalNode evalNode: evalNodes) { @@ -865,6 +868,10 @@ public Pair getIndexablePredicateValue(Set evalNodes) { } } + if (endDatum != null && columnMapping != null && columnMapping.getNumRowKeys() > 1) { + endDatum = new TextDatum(endDatum.asChars() + + new String(new char[]{columnMapping.getRowKeyDelimiter(), Character.MAX_VALUE})); + } if (startDatum != null || endDatum != null) { return new Pair(startDatum, endDatum); } else { diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java index 2a0ef33dc0..1fc40655f7 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java @@ -52,7 +52,7 @@ public void testGetIndexPredications() throws Exception { List> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); assertNotNull(indexEvals); assertEquals(1, indexEvals.size()); - Pair indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + Pair indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); assertEquals("020", indexPredicateValue.getFirst().asChars()); assertEquals("055", indexPredicateValue.getSecond().asChars()); @@ -62,11 +62,11 @@ public void testGetIndexPredications() throws Exception { scanNode.setQual(evalNodeB); indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); assertEquals(2, indexEvals.size()); - indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); assertEquals("020", indexPredicateValue.getFirst().asChars()); assertEquals("055", indexPredicateValue.getSecond().asChars()); - indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(1)); + indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1)); assertEquals("075", indexPredicateValue.getFirst().asChars()); assertEquals("075", indexPredicateValue.getSecond().asChars()); @@ -79,11 +79,11 @@ public void testGetIndexPredications() throws Exception { indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); assertEquals(2, indexEvals.size()); - indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); assertEquals("020", indexPredicateValue.getFirst().asChars()); assertEquals("055", indexPredicateValue.getSecond().asChars()); - indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(1)); + indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1)); assertEquals("072", indexPredicateValue.getFirst().asChars()); assertEquals("078", indexPredicateValue.getSecond().asChars()); @@ -98,11 +98,11 @@ public void testGetIndexPredications() throws Exception { indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); assertEquals(2, indexEvals.size()); - indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(0)); + indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0)); assertEquals("020", indexPredicateValue.getFirst().asChars()); assertEquals("055", indexPredicateValue.getSecond().asChars()); - indexPredicateValue = storageManager.getIndexablePredicateValue(indexEvals.get(1)); + indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1)); assertEquals("073", indexPredicateValue.getFirst().asChars()); assertEquals("078", indexPredicateValue.getSecond().asChars()); } From 97c32e41eb178af1138b7e6f822c3948d3063882 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Tue, 4 Nov 2014 16:49:03 +0900 Subject: [PATCH 20/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Implements Insert into location using hbase --- .../org/apache/tajo/master/GlobalEngine.java | 12 +- .../apache/tajo/master/querymaster/Query.java | 3 +- .../tajo/engine/query/TestHBaseTable.java | 73 +++- ...InsertIntoMultiRegionMultiRowFields.result | 200 +++++------ .../apache/tajo/plan/util/PlannerUtil.java | 30 +- .../tajo/storage/FileStorageManager.java | 297 +--------------- .../apache/tajo/storage/StorageManager.java | 326 +++++++++++++++++- .../apache/tajo/storage/TupleComparator.java | 17 - .../storage/hbase/HBaseStorageManager.java | 69 ++-- .../tajo/storage/hbase/HFileAppender.java | 5 +- .../src/main/resources/storage-default.xml | 4 + 11 files changed, 561 insertions(+), 475 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 174bbe07fd..ca4b3f114e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -303,7 +303,7 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, if (!storageProperty.isSupportsInsertInto()) { throw new VerifyException("Inserting into non-file storage is not supported."); } - sm.beforeCATS(rootNode.getChild()); + sm.beforeInsertOrCATS(rootNode.getChild()); } context.getSystemMetrics().counter("Query", "numDMLQuery").inc(); hookManager.doHooks(queryContext, plan); @@ -539,13 +539,11 @@ private void verifyInsertTableSchema(QueryContext queryContext, VerificationStat if (storeType != null) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); if (rootNode.getChild().getType() == NodeType.INSERT) { - String tableName = PlannerUtil.getStoreTableName(plan); - TableDesc tableDesc = catalog.getTableDesc(tableName); - - InsertNode iNode = rootNode.getChild(); - Schema outSchema = iNode.getChild().getOutSchema(); - try { + TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); + InsertNode iNode = rootNode.getChild(); + Schema outSchema = iNode.getChild().getOutSchema(); + StorageManager.getStorageManager(queryContext.getConf(), storeType) .verifyInsertTableSchema(tableDesc, outSchema); } catch (Throwable t) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index 3802060f99..c15c4cb498 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -412,7 +412,8 @@ private boolean finalizeQuery(Query query, QueryCompletedEvent event) { TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); Path finalOutputDir = StorageManager.getStorageManager(query.systemConf, storeType) - .commitOutputData(query.context.getQueryContext(), lastStage.getId(), lastStage.getSchema(), tableDesc); + .commitOutputData(query.context.getQueryContext(), + lastStage.getId(), lastStage.getMasterPlan().getLogicalPlan(), lastStage.getSchema(), tableDesc); QueryHookExecutor hookExecutor = new QueryHookExecutor(query.context.getQueryMasterContext()); hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(), finalOutputDir); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 1a80fb1b6c..052462a8ee 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -20,6 +20,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.Filter; @@ -965,7 +968,7 @@ public void testInsertIntoMultiRegionMultiRowFields() throws Exception { executeString("CREATE TABLE hbase_mapped_table (rk1 text, rk2 text, col1 text) " + "USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a', " + - "'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " + + "'hbase.split.rowkeys'='001,002,003,004,005,006,007,008,009', " + "'hbase.rowkey.delimiter'='_', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); @@ -982,11 +985,10 @@ public void testInsertIntoMultiRegionMultiRowFields() throws Exception { schema.addColumn("id1", Type.TEXT); schema.addColumn("id2", Type.TEXT); schema.addColumn("name", Type.TEXT); - //DecimalFormat df = new DecimalFormat("000"); + DecimalFormat df = new DecimalFormat("000"); List datas = new ArrayList(); for (int i = 99; i >= 0; i--) { -// datas.add(df.format(i) + "|" + (i + 100) + "|value" + i); - datas.add(i + "|" + (i + 100) + "|value" + i); + datas.add(df.format(i) + "|" + (i + 100) + "|value" + i); } TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", schema, tableOptions, datas.toArray(new String[]{}), 2); @@ -1287,6 +1289,69 @@ public void testInsertIntoUsingPut() throws Exception { } } + @Test + public void testInsertIntoLocation() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:', " + + "'hbase.split.rowkeys'='010,040,060,080', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + + try { + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", Type.TEXT); + schema.addColumn("name", Type.TEXT); + schema.addColumn("comment", Type.TEXT); + List datas = new ArrayList(); + DecimalFormat df = new DecimalFormat("000"); + for (int i = 99; i >= 0; i--) { + datas.add(df.format(i) + "|value" + i + "|comment-" + i); + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("insert into location '/tmp/hfile_test' " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')" + + "select id, name, comment from base_table ").close(); + + FileSystem fs = testingCluster.getDefaultFileSystem(); + Path path = new Path("/tmp/hfile_test"); + assertTrue(fs.exists(path)); + + FileStatus[] files = fs.listStatus(path); + assertNotNull(files); + assertEquals(2, files.length); + + assertEquals("/tmp/hfile_test/col2", files[1].getPath().toUri().getPath()); + + int index = 1; + for (FileStatus eachFile: files) { + assertEquals("/tmp/hfile_test/col" + index, eachFile.getPath().toUri().getPath()); + for (FileStatus subFile: fs.listStatus(eachFile.getPath())) { + assertTrue(subFile.isFile()); + assertTrue(subFile.getLen() > 0); + } + index++; + } + } finally { + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + } + } + private String resultSetToString(ResultScanner scanner, byte[][] cfNames, byte[][] qualifiers, boolean[] binaries, diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result index 1e6748a8da..0b740c8092 100644 --- a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result @@ -1,100 +1,100 @@ -0_100, value0 -10_110, value10 -11_111, value11 -12_112, value12 -13_113, value13 -14_114, value14 -15_115, value15 -16_116, value16 -17_117, value17 -18_118, value18 -19_119, value19 -1_101, value1 -20_120, value20 -21_121, value21 -22_122, value22 -23_123, value23 -24_124, value24 -25_125, value25 -26_126, value26 -27_127, value27 -28_128, value28 -29_129, value29 -2_102, value2 -30_130, value30 -31_131, value31 -32_132, value32 -33_133, value33 -34_134, value34 -35_135, value35 -36_136, value36 -37_137, value37 -38_138, value38 -39_139, value39 -3_103, value3 -40_140, value40 -41_141, value41 -42_142, value42 -43_143, value43 -44_144, value44 -45_145, value45 -46_146, value46 -47_147, value47 -48_148, value48 -49_149, value49 -4_104, value4 -50_150, value50 -51_151, value51 -52_152, value52 -53_153, value53 -54_154, value54 -55_155, value55 -56_156, value56 -57_157, value57 -58_158, value58 -59_159, value59 -5_105, value5 -60_160, value60 -61_161, value61 -62_162, value62 -63_163, value63 -64_164, value64 -65_165, value65 -66_166, value66 -67_167, value67 -68_168, value68 -69_169, value69 -6_106, value6 -70_170, value70 -71_171, value71 -72_172, value72 -73_173, value73 -74_174, value74 -75_175, value75 -76_176, value76 -77_177, value77 -78_178, value78 -79_179, value79 -7_107, value7 -80_180, value80 -81_181, value81 -82_182, value82 -83_183, value83 -84_184, value84 -85_185, value85 -86_186, value86 -87_187, value87 -88_188, value88 -89_189, value89 -8_108, value8 -90_190, value90 -91_191, value91 -92_192, value92 -93_193, value93 -94_194, value94 -95_195, value95 -96_196, value96 -97_197, value97 -98_198, value98 -99_199, value99 -9_109, value9 +000_100, value0 +001_101, value1 +002_102, value2 +003_103, value3 +004_104, value4 +005_105, value5 +006_106, value6 +007_107, value7 +008_108, value8 +009_109, value9 +010_110, value10 +011_111, value11 +012_112, value12 +013_113, value13 +014_114, value14 +015_115, value15 +016_116, value16 +017_117, value17 +018_118, value18 +019_119, value19 +020_120, value20 +021_121, value21 +022_122, value22 +023_123, value23 +024_124, value24 +025_125, value25 +026_126, value26 +027_127, value27 +028_128, value28 +029_129, value29 +030_130, value30 +031_131, value31 +032_132, value32 +033_133, value33 +034_134, value34 +035_135, value35 +036_136, value36 +037_137, value37 +038_138, value38 +039_139, value39 +040_140, value40 +041_141, value41 +042_142, value42 +043_143, value43 +044_144, value44 +045_145, value45 +046_146, value46 +047_147, value47 +048_148, value48 +049_149, value49 +050_150, value50 +051_151, value51 +052_152, value52 +053_153, value53 +054_154, value54 +055_155, value55 +056_156, value56 +057_157, value57 +058_158, value58 +059_159, value59 +060_160, value60 +061_161, value61 +062_162, value62 +063_163, value63 +064_164, value64 +065_165, value65 +066_166, value66 +067_167, value67 +068_168, value68 +069_169, value69 +070_170, value70 +071_171, value71 +072_172, value72 +073_173, value73 +074_174, value74 +075_175, value75 +076_176, value76 +077_177, value77 +078_178, value78 +079_179, value79 +080_180, value80 +081_181, value81 +082_182, value82 +083_183, value83 +084_184, value84 +085_185, value85 +086_186, value86 +087_187, value87 +088_188, value88 +089_189, value89 +090_190, value90 +091_191, value91 +092_192, value92 +093_193, value93 +094_194, value94 +095_195, value95 +096_196, value96 +097_197, value97 +098_198, value98 +099_199, value99 diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index ee13677871..74942e4b49 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -821,10 +821,10 @@ public static TableDesc getTableDesc(CatalogService catalog, LogicalNode node) t return createTableDesc((CreateTableNode)node); } String tableName = null; - if (node.getType() == NodeType.CREATE_TABLE) { - tableName = ((CreateTableNode)node).getTableName(); - } else if (node.getType() == NodeType.INSERT) { - tableName = ((InsertNode)node).getTableName(); + InsertNode insertNode = null; + if (node.getType() == NodeType.INSERT) { + insertNode = (InsertNode)node; + tableName = insertNode.getTableName(); } else { return null; } @@ -836,6 +836,11 @@ public static TableDesc getTableDesc(CatalogService catalog, LogicalNode node) t return catalog.getTableDesc(tableTokens[0], tableTokens[1]); } } + } else { + if (insertNode.getPath() != null) { + //insert ... location + return createTableDesc(insertNode); + } } return null; } @@ -858,4 +863,21 @@ private static TableDesc createTableDesc(CreateTableNode createTableNode) { return tableDescTobeCreated; } + + private static TableDesc createTableDesc(InsertNode insertNode) { + TableMeta meta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions()); + + TableDesc tableDescTobeCreated = + new TableDesc( + insertNode.getTableName(), + insertNode.getTableSchema(), + meta, + insertNode.getPath()); + + if (insertNode.hasPartition()) { + tableDescTobeCreated.setPartitionMethod(insertNode.getPartitionMethod()); + } + + return tableDescTobeCreated; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 03d024821f..019d1930a4 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -818,302 +818,7 @@ public void closeStorageManager() { } @Override - public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, Schema schema, - TableDesc tableDesc) { - Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); - Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); - Path finalOutputDir; - if (!queryContext.get(QueryVars.OUTPUT_TABLE_PATH, "").isEmpty()) { - finalOutputDir = new Path(queryContext.get(QueryVars.OUTPUT_TABLE_PATH)); - try { - FileSystem fs = stagingResultDir.getFileSystem(conf); - - if (queryContext.getBool(QueryVars.OUTPUT_OVERWRITE, false)) { // INSERT OVERWRITE INTO - - // It moves the original table into the temporary location. - // Then it moves the new result table into the original table location. - // Upon failed, it recovers the original table if possible. - boolean movedToOldTable = false; - boolean committed = false; - Path oldTableDir = new Path(stagingDir, TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); - - if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { - // This is a map for existing non-leaf directory to rename. A key is current directory and a value is - // renaming directory. - Map renameDirs = TUtil.newHashMap(); - // This is a map for recovering existing partition directory. A key is current directory and a value is - // temporary directory to back up. - Map recoveryDirs = TUtil.newHashMap(); - - try { - if (!fs.exists(finalOutputDir)) { - fs.mkdirs(finalOutputDir); - } - - visitPartitionedDirectory(fs, stagingResultDir, finalOutputDir, stagingResultDir.toString(), - renameDirs, oldTableDir); - - // Rename target partition directories - for(Map.Entry entry : renameDirs.entrySet()) { - // Backup existing data files for recovering - if (fs.exists(entry.getValue())) { - String recoveryPathString = entry.getValue().toString().replaceAll(finalOutputDir.toString(), - oldTableDir.toString()); - Path recoveryPath = new Path(recoveryPathString); - fs.rename(entry.getValue(), recoveryPath); - fs.exists(recoveryPath); - recoveryDirs.put(entry.getValue(), recoveryPath); - } - // Delete existing directory - fs.delete(entry.getValue(), true); - // Rename staging directory to final output directory - fs.rename(entry.getKey(), entry.getValue()); - } - - } catch (IOException ioe) { - // Remove created dirs - for(Map.Entry entry : renameDirs.entrySet()) { - fs.delete(entry.getValue(), true); - } - - // Recovery renamed dirs - for(Map.Entry entry : recoveryDirs.entrySet()) { - fs.delete(entry.getValue(), true); - fs.rename(entry.getValue(), entry.getKey()); - } - throw new IOException(ioe.getMessage()); - } - } else { - try { - if (fs.exists(finalOutputDir)) { - fs.rename(finalOutputDir, oldTableDir); - movedToOldTable = fs.exists(oldTableDir); - } else { // if the parent does not exist, make its parent directory. - fs.mkdirs(finalOutputDir.getParent()); - } - - fs.rename(stagingResultDir, finalOutputDir); - committed = fs.exists(finalOutputDir); - } catch (IOException ioe) { - // recover the old table - if (movedToOldTable && !committed) { - fs.rename(oldTableDir, finalOutputDir); - } - } - } - } else { - String queryType = queryContext.get(QueryVars.COMMAND_TYPE); - - if (queryType != null && queryType.equals(NodeType.INSERT.name())) { // INSERT INTO an existing table - - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(3); - - if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { - for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { - if (eachFile.isFile()) { - LOG.warn("Partition table can't have file in a staging dir: " + eachFile.getPath()); - continue; - } - moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, -1); - } - } else { - int maxSeq = StorageUtil.getMaxFileSequence(fs, finalOutputDir, false) + 1; - for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { - moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, maxSeq++); - } - } - // checking all file moved and remove empty dir - verifyAllFileMoved(fs, stagingResultDir); - FileStatus[] files = fs.listStatus(stagingResultDir); - if (files != null && files.length != 0) { - for (FileStatus eachFile: files) { - LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); - } - } - } else { // CREATE TABLE AS SELECT (CTAS) - fs.rename(stagingResultDir, finalOutputDir); - LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); - } - } - } catch (IOException e) { - // TODO report to client - e.printStackTrace(); - } - } else { - finalOutputDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); - } - - return finalOutputDir; - } - - /** - * This method sets a rename map which includes renamed staging directory to final output directory recursively. - * If there exists some data files, this delete it for duplicate data. - * - * - * @param fs - * @param stagingPath - * @param outputPath - * @param stagingParentPathString - * @throws IOException - */ - private void visitPartitionedDirectory(FileSystem fs, Path stagingPath, Path outputPath, - String stagingParentPathString, - Map renameDirs, Path oldTableDir) throws IOException { - FileStatus[] files = fs.listStatus(stagingPath); - - for(FileStatus eachFile : files) { - if (eachFile.isDirectory()) { - Path oldPath = eachFile.getPath(); - - // Make recover directory. - String recoverPathString = oldPath.toString().replaceAll(stagingParentPathString, - oldTableDir.toString()); - Path recoveryPath = new Path(recoverPathString); - if (!fs.exists(recoveryPath)) { - fs.mkdirs(recoveryPath); - } - - visitPartitionedDirectory(fs, eachFile.getPath(), outputPath, stagingParentPathString, - renameDirs, oldTableDir); - // Find last order partition for renaming - String newPathString = oldPath.toString().replaceAll(stagingParentPathString, - outputPath.toString()); - Path newPath = new Path(newPathString); - if (!isLeafDirectory(fs, eachFile.getPath())) { - renameDirs.put(eachFile.getPath(), newPath); - } else { - if (!fs.exists(newPath)) { - fs.mkdirs(newPath); - } - } - } - } - } - - private boolean isLeafDirectory(FileSystem fs, Path path) throws IOException { - boolean retValue = false; - - FileStatus[] files = fs.listStatus(path); - for (FileStatus file : files) { - if (fs.isDirectory(file.getPath())) { - retValue = true; - break; - } - } - - return retValue; - } - - private boolean verifyAllFileMoved(FileSystem fs, Path stagingPath) throws IOException { - FileStatus[] files = fs.listStatus(stagingPath); - if (files != null && files.length != 0) { - for (FileStatus eachFile: files) { - if (eachFile.isFile()) { - LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); - return false; - } else { - if (verifyAllFileMoved(fs, eachFile.getPath())) { - fs.delete(eachFile.getPath(), false); - } else { - return false; - } - } - } - } - - return true; - } - - /** - * Attach the sequence number to the output file name and than move the file into the final result path. - * - * @param fs FileSystem - * @param stagingResultDir The staging result dir - * @param fileStatus The file status - * @param finalOutputPath Final output path - * @param nf Number format - * @param fileSeq The sequence number - * @throws IOException - */ - private void moveResultFromStageToFinal(FileSystem fs, Path stagingResultDir, - FileStatus fileStatus, Path finalOutputPath, - NumberFormat nf, - int fileSeq) throws IOException { - if (fileStatus.isDirectory()) { - String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); - if (subPath != null) { - Path finalSubPath = new Path(finalOutputPath, subPath); - if (!fs.exists(finalSubPath)) { - fs.mkdirs(finalSubPath); - } - int maxSeq = StorageUtil.getMaxFileSequence(fs, finalSubPath, false); - for (FileStatus eachFile : fs.listStatus(fileStatus.getPath())) { - moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputPath, nf, ++maxSeq); - } - } else { - throw new IOException("Wrong staging dir:" + stagingResultDir + "," + fileStatus.getPath()); - } - } else { - String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); - if (subPath != null) { - Path finalSubPath = new Path(finalOutputPath, subPath); - finalSubPath = new Path(finalSubPath.getParent(), replaceFileNameSeq(finalSubPath, fileSeq, nf)); - if (!fs.exists(finalSubPath.getParent())) { - fs.mkdirs(finalSubPath.getParent()); - } - if (fs.exists(finalSubPath)) { - throw new IOException("Already exists data file:" + finalSubPath); - } - boolean success = fs.rename(fileStatus.getPath(), finalSubPath); - if (success) { - LOG.info("Moving staging file[" + fileStatus.getPath() + "] + " + - "to final output[" + finalSubPath + "]"); - } else { - LOG.error("Can't move staging file[" + fileStatus.getPath() + "] + " + - "to final output[" + finalSubPath + "]"); - } - } - } - } - - private String extractSubPath(Path parentPath, Path childPath) { - String parentPathStr = parentPath.toUri().getPath(); - String childPathStr = childPath.toUri().getPath(); - - if (parentPathStr.length() > childPathStr.length()) { - return null; - } - - int index = childPathStr.indexOf(parentPathStr); - if (index != 0) { - return null; - } - - return childPathStr.substring(parentPathStr.length() + 1); - } - - /** - * Attach the sequence number to a path. - * - * @param path Path - * @param seq sequence number - * @param nf Number format - * @return New path attached with sequence number - * @throws IOException - */ - private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IOException { - String[] tokens = path.getName().split("-"); - if (tokens.length != 4) { - throw new IOException("Wrong result file name:" + path); - } - return tokens[0] + "-" + tokens[1] + "-" + tokens[2] + "-" + nf.format(seq); - } - - @Override - public void beforeCATS(LogicalNode node) throws IOException { + public void beforeInsertOrCATS(LogicalNode node) throws IOException { } @Override diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 6736e49426..8b423bc2c2 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -23,26 +23,26 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; -import org.apache.tajo.ExecutionBlockId; -import org.apache.tajo.OverridableConf; -import org.apache.tajo.QueryUnitAttemptId; -import org.apache.tajo.TajoConstants; +import org.apache.tajo.*; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.storage.hbase.HBaseStorageManager; +import org.apache.tajo.util.TUtil; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.URI; +import java.text.NumberFormat; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -132,18 +132,6 @@ public abstract List getNonForwardSplit(TableDesc tableDesc, int curre */ public abstract void closeStorageManager(); - /** - * It moves a result data stored in a staging output dir into a final output dir. - * @param queryContext - * @param finalEbId - * @param schema - * @param tableDesc - * @return - * @throws IOException - */ - public abstract Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, Schema schema, - TableDesc tableDesc) throws IOException; - /** * * @param queryContext @@ -161,7 +149,7 @@ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, T * @param node * @throws IOException */ - public abstract void beforeCATS(LogicalNode node) throws IOException; + public abstract void beforeInsertOrCATS(LogicalNode node) throws IOException; /** * @@ -395,4 +383,308 @@ public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throw public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { return null; } + + public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, + LogicalPlan plan, Schema schema, + TableDesc tableDesc) throws IOException { + return commitOutputData(queryContext, finalEbId, plan, schema, tableDesc, true); + } + + protected Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, + LogicalPlan plan, Schema schema, + TableDesc tableDesc, boolean changeFileSeq) throws IOException { + Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); + Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); + Path finalOutputDir; + if (!queryContext.get(QueryVars.OUTPUT_TABLE_PATH, "").isEmpty()) { + finalOutputDir = new Path(queryContext.get(QueryVars.OUTPUT_TABLE_PATH)); + FileSystem fs = stagingResultDir.getFileSystem(conf); + + if (queryContext.getBool(QueryVars.OUTPUT_OVERWRITE, false)) { // INSERT OVERWRITE INTO + + // It moves the original table into the temporary location. + // Then it moves the new result table into the original table location. + // Upon failed, it recovers the original table if possible. + boolean movedToOldTable = false; + boolean committed = false; + Path oldTableDir = new Path(stagingDir, TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); + + if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { + // This is a map for existing non-leaf directory to rename. A key is current directory and a value is + // renaming directory. + Map renameDirs = TUtil.newHashMap(); + // This is a map for recovering existing partition directory. A key is current directory and a value is + // temporary directory to back up. + Map recoveryDirs = TUtil.newHashMap(); + + try { + if (!fs.exists(finalOutputDir)) { + fs.mkdirs(finalOutputDir); + } + + visitPartitionedDirectory(fs, stagingResultDir, finalOutputDir, stagingResultDir.toString(), + renameDirs, oldTableDir); + + // Rename target partition directories + for(Map.Entry entry : renameDirs.entrySet()) { + // Backup existing data files for recovering + if (fs.exists(entry.getValue())) { + String recoveryPathString = entry.getValue().toString().replaceAll(finalOutputDir.toString(), + oldTableDir.toString()); + Path recoveryPath = new Path(recoveryPathString); + fs.rename(entry.getValue(), recoveryPath); + fs.exists(recoveryPath); + recoveryDirs.put(entry.getValue(), recoveryPath); + } + // Delete existing directory + fs.delete(entry.getValue(), true); + // Rename staging directory to final output directory + fs.rename(entry.getKey(), entry.getValue()); + } + + } catch (IOException ioe) { + // Remove created dirs + for(Map.Entry entry : renameDirs.entrySet()) { + fs.delete(entry.getValue(), true); + } + + // Recovery renamed dirs + for(Map.Entry entry : recoveryDirs.entrySet()) { + fs.delete(entry.getValue(), true); + fs.rename(entry.getValue(), entry.getKey()); + } + throw new IOException(ioe.getMessage()); + } + } else { + try { + if (fs.exists(finalOutputDir)) { + fs.rename(finalOutputDir, oldTableDir); + movedToOldTable = fs.exists(oldTableDir); + } else { // if the parent does not exist, make its parent directory. + fs.mkdirs(finalOutputDir.getParent()); + } + + fs.rename(stagingResultDir, finalOutputDir); + committed = fs.exists(finalOutputDir); + } catch (IOException ioe) { + // recover the old table + if (movedToOldTable && !committed) { + fs.rename(oldTableDir, finalOutputDir); + } + } + } + } else { + String queryType = queryContext.get(QueryVars.COMMAND_TYPE); + + if (queryType != null && queryType.equals(NodeType.INSERT.name())) { // INSERT INTO an existing table + + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(3); + + if (!queryContext.get(QueryVars.OUTPUT_PARTITIONS, "").isEmpty()) { + for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { + if (eachFile.isFile()) { + LOG.warn("Partition table can't have file in a staging dir: " + eachFile.getPath()); + continue; + } + moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, -1, changeFileSeq); + } + } else { + int maxSeq = StorageUtil.getMaxFileSequence(fs, finalOutputDir, false) + 1; + for(FileStatus eachFile: fs.listStatus(stagingResultDir)) { + if (eachFile.getPath().getName().startsWith("_")) { + continue; + } + moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputDir, fmt, maxSeq++, changeFileSeq); + } + } + // checking all file moved and remove empty dir + verifyAllFileMoved(fs, stagingResultDir); + FileStatus[] files = fs.listStatus(stagingResultDir); + if (files != null && files.length != 0) { + for (FileStatus eachFile: files) { + LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); + } + } + } else { // CREATE TABLE AS SELECT (CTAS) + fs.rename(stagingResultDir, finalOutputDir); + LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir); + } + } + } else { + finalOutputDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); + } + + return finalOutputDir; + } + + /** + * Attach the sequence number to the output file name and than move the file into the final result path. + * + * @param fs FileSystem + * @param stagingResultDir The staging result dir + * @param fileStatus The file status + * @param finalOutputPath Final output path + * @param nf Number format + * @param fileSeq The sequence number + * @throws IOException + */ + private void moveResultFromStageToFinal(FileSystem fs, Path stagingResultDir, + FileStatus fileStatus, Path finalOutputPath, + NumberFormat nf, + int fileSeq, boolean changeFileSeq) throws IOException { + if (fileStatus.isDirectory()) { + String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); + if (subPath != null) { + Path finalSubPath = new Path(finalOutputPath, subPath); + if (!fs.exists(finalSubPath)) { + fs.mkdirs(finalSubPath); + } + int maxSeq = StorageUtil.getMaxFileSequence(fs, finalSubPath, false); + for (FileStatus eachFile : fs.listStatus(fileStatus.getPath())) { + if (eachFile.getPath().getName().startsWith("_")) { + continue; + } + moveResultFromStageToFinal(fs, stagingResultDir, eachFile, finalOutputPath, nf, ++maxSeq, changeFileSeq); + } + } else { + throw new IOException("Wrong staging dir:" + stagingResultDir + "," + fileStatus.getPath()); + } + } else { + String subPath = extractSubPath(stagingResultDir, fileStatus.getPath()); + if (subPath != null) { + Path finalSubPath = new Path(finalOutputPath, subPath); + if (changeFileSeq) { + finalSubPath = new Path(finalSubPath.getParent(), replaceFileNameSeq(finalSubPath, fileSeq, nf)); + } + if (!fs.exists(finalSubPath.getParent())) { + fs.mkdirs(finalSubPath.getParent()); + } + if (fs.exists(finalSubPath)) { + throw new IOException("Already exists data file:" + finalSubPath); + } + boolean success = fs.rename(fileStatus.getPath(), finalSubPath); + if (success) { + LOG.info("Moving staging file[" + fileStatus.getPath() + "] + " + + "to final output[" + finalSubPath + "]"); + } else { + LOG.error("Can't move staging file[" + fileStatus.getPath() + "] + " + + "to final output[" + finalSubPath + "]"); + } + } + } + } + + private String extractSubPath(Path parentPath, Path childPath) { + String parentPathStr = parentPath.toUri().getPath(); + String childPathStr = childPath.toUri().getPath(); + + if (parentPathStr.length() > childPathStr.length()) { + return null; + } + + int index = childPathStr.indexOf(parentPathStr); + if (index != 0) { + return null; + } + + return childPathStr.substring(parentPathStr.length() + 1); + } + + /** + * Attach the sequence number to a path. + * + * @param path Path + * @param seq sequence number + * @param nf Number format + * @return New path attached with sequence number + * @throws IOException + */ + private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IOException { + String[] tokens = path.getName().split("-"); + if (tokens.length != 4) { + throw new IOException("Wrong result file name:" + path); + } + return tokens[0] + "-" + tokens[1] + "-" + tokens[2] + "-" + nf.format(seq); + } + + private boolean verifyAllFileMoved(FileSystem fs, Path stagingPath) throws IOException { + FileStatus[] files = fs.listStatus(stagingPath); + if (files != null && files.length != 0) { + for (FileStatus eachFile: files) { + if (eachFile.isFile()) { + LOG.error("There are some unmoved files in staging dir:" + eachFile.getPath()); + return false; + } else { + if (verifyAllFileMoved(fs, eachFile.getPath())) { + fs.delete(eachFile.getPath(), false); + } else { + return false; + } + } + } + } + + return true; + } + + /** + * This method sets a rename map which includes renamed staging directory to final output directory recursively. + * If there exists some data files, this delete it for duplicate data. + * + * + * @param fs + * @param stagingPath + * @param outputPath + * @param stagingParentPathString + * @throws IOException + */ + private void visitPartitionedDirectory(FileSystem fs, Path stagingPath, Path outputPath, + String stagingParentPathString, + Map renameDirs, Path oldTableDir) throws IOException { + FileStatus[] files = fs.listStatus(stagingPath); + + for(FileStatus eachFile : files) { + if (eachFile.isDirectory()) { + Path oldPath = eachFile.getPath(); + + // Make recover directory. + String recoverPathString = oldPath.toString().replaceAll(stagingParentPathString, + oldTableDir.toString()); + Path recoveryPath = new Path(recoverPathString); + if (!fs.exists(recoveryPath)) { + fs.mkdirs(recoveryPath); + } + + visitPartitionedDirectory(fs, eachFile.getPath(), outputPath, stagingParentPathString, + renameDirs, oldTableDir); + // Find last order partition for renaming + String newPathString = oldPath.toString().replaceAll(stagingParentPathString, + outputPath.toString()); + Path newPath = new Path(newPathString); + if (!isLeafDirectory(fs, eachFile.getPath())) { + renameDirs.put(eachFile.getPath(), newPath); + } else { + if (!fs.exists(newPath)) { + fs.mkdirs(newPath); + } + } + } + } + } + + private boolean isLeafDirectory(FileSystem fs, Path path) throws IOException { + boolean retValue = false; + + FileStatus[] files = fs.listStatus(path); + for (FileStatus file : files) { + if (fs.isDirectory(file.getPath())) { + retValue = true; + break; + } + } + + return retValue; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java index c3a5ed47c8..084c105d2b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java @@ -88,23 +88,6 @@ public int compare(Tuple tuple1, Tuple tuple2) { Datum right = null; int compVal = 0; - String sb1 = ""; - String sb2 = ""; - for (int i = 0; i < sortKeyIds.length; i++) { - left = tuple1.get(sortKeyIds[i]); - right = tuple2.get(sortKeyIds[i]); - if (i > 0) { - sb1 += "_" + left.asChars(); - sb2 += "_" + right.asChars(); - } else { - sb1 += "_" + left.asChars(); - sb2 += "_" + right.asChars(); - } - } - if (1==1) { - return sb1.compareTo(sb2); - } - for (int i = 0; i < sortKeyIds.length; i++) { left = tuple1.get(sortKeyIds[i]); right = tuple2.get(sortKeyIds[i]); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 1f4cc5a417..245e4f38d9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -40,6 +40,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.RewriteRule; @@ -324,6 +325,7 @@ public void purgeTable(TableDesc tableDesc) throws IOException { try { HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc.getMeta(), tableDesc.getSchema()); + LOG.info("Deleting hbase table: " + new String(hTableDesc.getName())); hAdmin.disableTable(hTableDesc.getName()); hAdmin.deleteTable(hTableDesc.getName()); } finally { @@ -880,12 +882,14 @@ public Pair getIndexablePredicateValue(ColumnMapping columnMapping } @Override - public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, Schema schema, + public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, + LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { if (tableDesc == null) { throw new IOException("TableDesc is null while calling loadIncrementalHFiles: " + finalEbId); } - Path finalOutputDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); + Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); + Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); hbaseConf.set("hbase.loadincremental.threads.max", "2"); @@ -893,31 +897,38 @@ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId fina JobContextImpl jobContext = new JobContextImpl(hbaseConf, new JobID(finalEbId.getQueryId().toString(), finalEbId.getId())); - FileOutputCommitter committer = new FileOutputCommitter(finalOutputDir, jobContext); + FileOutputCommitter committer = new FileOutputCommitter(stagingResultDir, jobContext); Path jobAttemptPath = committer.getJobAttemptPath(jobContext); FileSystem fs = jobAttemptPath.getFileSystem(queryContext.getConf()); if (!fs.exists(jobAttemptPath) || fs.listStatus(jobAttemptPath) == null) { LOG.warn("No query attempt file in " + jobAttemptPath); - return finalOutputDir; + return stagingResultDir; } committer.commitJob(jobContext); - String tableName = tableDesc.getMeta().getOption(HBaseStorageManager.META_TABLE_KEY); + if (tableDesc.getName() == null && tableDesc.getPath() != null) { - HTable htable = new HTable(hbaseConf, tableName); - try { - LoadIncrementalHFiles loadIncrementalHFiles = null; + // insert into location + return super.commitOutputData(queryContext, finalEbId, plan, schema, tableDesc, false); + } else { + // insert into table + String tableName = tableDesc.getMeta().getOption(HBaseStorageManager.META_TABLE_KEY); + + HTable htable = new HTable(hbaseConf, tableName); try { - loadIncrementalHFiles = new LoadIncrementalHFiles(hbaseConf); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new IOException(e.getMessage(), e); - } - loadIncrementalHFiles.doBulkLoad(finalOutputDir, htable); + LoadIncrementalHFiles loadIncrementalHFiles = null; + try { + loadIncrementalHFiles = new LoadIncrementalHFiles(hbaseConf); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new IOException(e.getMessage(), e); + } + loadIncrementalHFiles.doBulkLoad(stagingResultDir, htable); - return finalOutputDir; - } finally { - htable.close(); + return stagingResultDir; + } finally { + htable.close(); + } } } @@ -1032,7 +1043,7 @@ public StorageProperty getStorageProperty() { return storageProperty; } - public void beforeCATS(LogicalNode node) throws IOException { + public void beforeInsertOrCATS(LogicalNode node) throws IOException { if (node.getType() == NodeType.CREATE_TABLE) { CreateTableNode cNode = (CreateTableNode)node; if (!cNode.isExternal()) { @@ -1065,17 +1076,19 @@ public void queryFailed(LogicalNode node) throws IOException { @Override public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { - Schema tableSchema = tableDesc.getSchema(); - if (tableSchema.size() != outSchema.size()) { - throw new IOException("The number of table columns is different from SELECT columns"); - } + if (tableDesc != null) { + Schema tableSchema = tableDesc.getSchema(); + if (tableSchema.size() != outSchema.size()) { + throw new IOException("The number of table columns is different from SELECT columns"); + } - for (int i = 0; i < tableSchema.size(); i++) { - if (!tableSchema.getColumn(i).getDataType().equals(outSchema.getColumn(i).getDataType())) { - throw new IOException(outSchema.getColumn(i).getQualifiedName() + - "(" + outSchema.getColumn(i).getDataType().getType() + ")" + - " is different column type with " + tableSchema.getColumn(i).getSimpleName() + - "(" + tableSchema.getColumn(i).getDataType().getType() + ")"); + for (int i = 0; i < tableSchema.size(); i++) { + if (!tableSchema.getColumn(i).getDataType().equals(outSchema.getColumn(i).getDataType())) { + throw new IOException(outSchema.getColumn(i).getQualifiedName() + + "(" + outSchema.getColumn(i).getDataType().getType() + ")" + + " is different column type with " + tableSchema.getColumn(i).getSimpleName() + + "(" + tableSchema.getColumn(i).getDataType().getType() + ")"); + } } } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java index 29f434c6a4..e2fce2b594 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java @@ -35,6 +35,8 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryUnitAttemptId; +import org.apache.tajo.QueryVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; @@ -69,7 +71,8 @@ public void init() throws IOException { super.init(); Configuration taskConf = new Configuration(); - taskConf.set(FileOutputFormat.OUTDIR, stagingDir.toString()); + Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); + taskConf.set(FileOutputFormat.OUTDIR, stagingResultDir.toString()); ExecutionBlockId ebId = taskAttemptId.getQueryUnitId().getExecutionBlockId(); writerContext = new TaskAttemptContextImpl(taskConf, diff --git a/tajo-storage/src/main/resources/storage-default.xml b/tajo-storage/src/main/resources/storage-default.xml index 6efd3186e7..efe7dd288f 100644 --- a/tajo-storage/src/main/resources/storage-default.xml +++ b/tajo-storage/src/main/resources/storage-default.xml @@ -161,4 +161,8 @@ org.apache.tajo.storage.hbase.HFileAppender + + tajo.storage.appender-handler.hfile.class + org.apache.tajo.storage.hbase.HFileAppender + From 402022f186192f0b1df504ec83b623f7a4f231b8 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Tue, 4 Nov 2014 17:14:57 +0900 Subject: [PATCH 21/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Fix hbase region splitting --- .../org/apache/tajo/QueryTestCaseBase.java | 6 +- .../tajo/engine/query/TestHBaseTable.java | 164 ++++++++++++------ .../dataset/TestHBaseTable/splits.data | 4 + ...tInsertIntoMultiRegionWithSplitFile.result | 100 +++++++++++ .../storage/hbase/HBaseStorageManager.java | 1 - 5 files changed, 221 insertions(+), 54 deletions(-) create mode 100644 tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 9431b26cb8..8e224a7310 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -168,9 +168,9 @@ public class QueryTestCaseBase { private static String currentDatabase; private static Set createdTableGlobalSet = new HashSet(); // queries and results directory corresponding to subclass class. - private Path currentQueryPath; - private Path currentResultPath; - private Path currentDatasetPath; + protected Path currentQueryPath; + protected Path currentResultPath; + protected Path currentDatasetPath; // for getting a method name @Rule public TestName name = new TestName(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 052462a8ee..b7e7405976 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -805,14 +805,14 @@ public void testInsertIntoMultiRegion() throws Exception { } @Test - public void testInsertIntoColumnKeyValue() throws Exception { + public void testInsertIntoMultiRegion2() throws Exception { String hostName = InetAddress.getLocalHost().getHostName(); String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); assertNotNull(zkPort); - executeString("CREATE TABLE hbase_mapped_table (rk text, col2_key text, col2_value text, col3 text) " + - "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " + - "'hbase.rowkey.delimiter'='_', " + + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " + + "'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); @@ -825,21 +825,17 @@ public void testInsertIntoColumnKeyValue() throws Exception { tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); Schema schema = new Schema(); - schema.addColumn("rk", Type.TEXT); - schema.addColumn("col2_key", Type.TEXT); - schema.addColumn("col2_value", Type.TEXT); - schema.addColumn("col3", Type.TEXT); + schema.addColumn("id", Type.TEXT); + schema.addColumn("name", Type.TEXT); List datas = new ArrayList(); - for (int i = 20; i >= 0; i--) { - for (int j = 0; j < 3; j++) { - datas.add(i + "|ck-" + j + "|value-" + j + "|col3-" + i); - } + for (int i = 99; i >= 0; i--) { + datas.add(i + "|value" + i); } TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", schema, tableOptions, datas.toArray(new String[]{}), 2); executeString("insert into hbase_mapped_table " + - "select rk, col2_key, col2_value, col3 from base_table ").close(); + "select id, name from base_table ").close(); HTable htable = null; ResultScanner scanner = null; @@ -847,43 +843,13 @@ public void testInsertIntoColumnKeyValue() throws Exception { htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); Scan scan = new Scan(); - scan.addFamily(Bytes.toBytes("col2")); - scan.addFamily(Bytes.toBytes("col3")); + scan.addFamily(Bytes.toBytes("col1")); scanner = htable.getScanner(scan); assertStrings(resultSetToString(scanner, - new byte[][]{null, Bytes.toBytes("col2"), Bytes.toBytes("col3")}, - new byte[][]{null, null, null}, - new boolean[]{false, false, false}, tableDesc.getSchema())); - - ResultSet res = executeString("select * from hbase_mapped_table"); - - String expected = "rk,col2_key,col2_value,col3\n" + - "-------------------------------\n" + - "0,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-0\n" + - "1,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-1\n" + - "10,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-10\n" + - "11,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-11\n" + - "12,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-12\n" + - "13,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-13\n" + - "14,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-14\n" + - "15,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-15\n" + - "16,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-16\n" + - "17,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-17\n" + - "18,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-18\n" + - "19,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-19\n" + - "2,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-2\n" + - "20,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-20\n" + - "3,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-3\n" + - "4,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-4\n" + - "5,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-5\n" + - "6,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-6\n" + - "7,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-7\n" + - "8,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-8\n" + - "9,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-9\n"; - - assertEquals(expected, resultSetToString(res)); - res.close(); + new byte[][]{null, Bytes.toBytes("col1")}, + new byte[][]{null, Bytes.toBytes("a")}, + new boolean[]{false, false}, tableDesc.getSchema())); } finally { executeString("DROP TABLE base_table PURGE").close(); @@ -900,14 +866,16 @@ public void testInsertIntoColumnKeyValue() throws Exception { } @Test - public void testInsertIntoMultiRegion2() throws Exception { + public void testInsertIntoMultiRegionWithSplitFile() throws Exception { String hostName = InetAddress.getLocalHost().getHostName(); String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); assertNotNull(zkPort); + String splitFilePath = currentDatasetPath + "/splits.data"; + executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " + - "'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " + + "'hbase.split.rowkeys.file'='" + splitFilePath + "', " + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); @@ -923,8 +891,9 @@ public void testInsertIntoMultiRegion2() throws Exception { schema.addColumn("id", Type.TEXT); schema.addColumn("name", Type.TEXT); List datas = new ArrayList(); + DecimalFormat df = new DecimalFormat("000"); for (int i = 99; i >= 0; i--) { - datas.add(i + "|value" + i); + datas.add(df.format(i) + "|value" + i); } TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", schema, tableOptions, datas.toArray(new String[]{}), 2); @@ -1085,6 +1054,101 @@ public void testInsertIntoBinaryMultiRegion() throws Exception { } } + @Test + public void testInsertIntoColumnKeyValue() throws Exception { + String hostName = InetAddress.getLocalHost().getHostName(); + String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT); + assertNotNull(zkPort); + + executeString("CREATE TABLE hbase_mapped_table (rk text, col2_key text, col2_value text, col3 text) " + + "USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " + + "'hbase.rowkey.delimiter'='_', " + + "'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," + + "'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close(); + + assertTableExists("hbase_mapped_table"); + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), "hbase_mapped_table"); + + // create test table + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("rk", Type.TEXT); + schema.addColumn("col2_key", Type.TEXT); + schema.addColumn("col2_value", Type.TEXT); + schema.addColumn("col3", Type.TEXT); + List datas = new ArrayList(); + for (int i = 20; i >= 0; i--) { + for (int j = 0; j < 3; j++) { + datas.add(i + "|ck-" + j + "|value-" + j + "|col3-" + i); + } + } + TajoTestingCluster.createTable(getCurrentDatabase() + ".base_table", + schema, tableOptions, datas.toArray(new String[]{}), 2); + + executeString("insert into hbase_mapped_table " + + "select rk, col2_key, col2_value, col3 from base_table ").close(); + + HTable htable = null; + ResultScanner scanner = null; + try { + htable = new HTable(testingCluster.getHBaseUtil().getConf(), "hbase_table"); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("col2")); + scan.addFamily(Bytes.toBytes("col3")); + scanner = htable.getScanner(scan); + + assertStrings(resultSetToString(scanner, + new byte[][]{null, Bytes.toBytes("col2"), Bytes.toBytes("col3")}, + new byte[][]{null, null, null}, + new boolean[]{false, false, false}, tableDesc.getSchema())); + + ResultSet res = executeString("select * from hbase_mapped_table"); + + String expected = "rk,col2_key,col2_value,col3\n" + + "-------------------------------\n" + + "0,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-0\n" + + "1,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-1\n" + + "10,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-10\n" + + "11,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-11\n" + + "12,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-12\n" + + "13,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-13\n" + + "14,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-14\n" + + "15,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-15\n" + + "16,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-16\n" + + "17,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-17\n" + + "18,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-18\n" + + "19,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-19\n" + + "2,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-2\n" + + "20,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-20\n" + + "3,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-3\n" + + "4,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-4\n" + + "5,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-5\n" + + "6,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-6\n" + + "7,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-7\n" + + "8,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-8\n" + + "9,[\"ck-0\", \"ck-1\", \"ck-2\"],[\"value-0\", \"value-1\", \"value-2\"],col3-9\n"; + + assertEquals(expected, resultSetToString(res)); + res.close(); + + } finally { + executeString("DROP TABLE base_table PURGE").close(); + executeString("DROP TABLE hbase_mapped_table PURGE").close(); + + if (scanner != null) { + scanner.close(); + } + + if (htable != null) { + htable.close(); + } + } + } + @Test public void testInsertIntoDifferentType() throws Exception { String hostName = InetAddress.getLocalHost().getHostName(); diff --git a/tajo-core/src/test/resources/dataset/TestHBaseTable/splits.data b/tajo-core/src/test/resources/dataset/TestHBaseTable/splits.data index e69de29bb2..417d480501 100644 --- a/tajo-core/src/test/resources/dataset/TestHBaseTable/splits.data +++ b/tajo-core/src/test/resources/dataset/TestHBaseTable/splits.data @@ -0,0 +1,4 @@ +010 +040 +060 +080 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result new file mode 100644 index 0000000000..72013f26e9 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99 diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 245e4f38d9..3a86520aa5 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -261,7 +261,6 @@ private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) thro byte[][] splitKeys = new byte[splitKeySet.size()][]; int index = 0; for (String eachKey: splitKeySet) { - splitKeys[index++] = Bytes.toBytes(eachKey); if (numRowKeys == 1 && rowkeyBinary) { splitKeys[index++] = HBaseBinarySerializerDeserializer.serialize(rowKeyColumn, new TextDatum(eachKey)); } else { From e156b1c642971e95f364c0668bfced96b567c600 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Tue, 4 Nov 2014 22:45:11 +0900 Subject: [PATCH 22/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Optimize HBase Scanner --- .../org/apache/tajo/master/GlobalEngine.java | 3 ++ .../tajo/storage/hbase/ColumnMapping.java | 10 ++--- .../tajo/storage/hbase/HBaseScanner.java | 45 ++++++++++++------- .../tajo/storage/hbase/TestColumnMapping.java | 8 ++++ 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index ca4b3f114e..fdbe6b3c92 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -252,6 +252,9 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT); maxRow = (int) limitNode.getFetchFirstNum(); } + if (desc.getStats().getNumRows() == 0) { + desc.getStats().setNumRows(-1); + } QueryId queryId = QueryIdFactory.newQueryId(context.getResourceManager().getSeedQueryId()); NonForwardQueryResultScanner queryResultScanner = diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java index b48bb22c53..e5bee1a2bf 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -93,7 +93,7 @@ public void init() throws IOException { if (mappingTokens.length == 3) { if (mappingTokens[0].length == 0) { // cfname - throw new IOException("'column' attribute should be ':key:' or ':key:#b' " + + throw new IOException(eachToken + " 'column' attribute should be ':key:' or ':key:#b' " + "or ':value:' or ':value:#b'"); } //:key: or :value: @@ -102,7 +102,7 @@ public void init() throws IOException { if ("#b".equals(binaryOption)) { isBinaryColumns[index] = true; } else { - throw new IOException("'column' attribute should be ':key:' or ':key:#b' " + + throw new IOException(eachToken + " 'column' attribute should be ':key:' or ':key:#b' " + "or ':value:' or ':value:#b'"); } } @@ -113,7 +113,7 @@ public void init() throws IOException { } else if (VALUE_COLUMN_MAPPING.equalsIgnoreCase(keyOrValue)) { isColumnValues[index] = true; } else { - throw new IOException("'column' attribute should be ':key:' or ':value:'"); + throw new IOException(eachToken + " 'column' attribute should be ':key:' or ':value:'"); } } else if (mappingTokens.length == 2) { //: or : or :key @@ -134,7 +134,7 @@ public void init() throws IOException { } } else { if (cfName.isEmpty()) { - throw new IOException("'column' attribute should be ':key:' or ':value:'"); + throw new IOException(eachToken + " 'column' attribute should be ':key:' or ':value:'"); } if (cfName != null) { mappingColumns[index][0] = Bytes.toBytes(cfName); @@ -153,7 +153,7 @@ public void init() throws IOException { } } } else { - throw new IOException("'column' attribute '[cfname]:[qualfier]:"); + throw new IOException(eachToken + " 'column' attribute '[cfname]:[qualfier]:'"); } index++; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index 7077b697ba..04e7caebe0 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -22,8 +22,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.*; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -137,27 +136,43 @@ public void init() throws IOException { private void initScanner() throws IOException { scan = new Scan(); scan.setBatch(scanFetchSize); + scan.setCacheBlocks(false); + scan.setCaching(scanFetchSize); + + FilterList filters = null; + if (targetIndexes == null || targetIndexes.length == 0) { + filters = new FilterList(FilterList.Operator.MUST_PASS_ALL); + filters.addFilter(new FirstKeyOnlyFilter()); + filters.addFilter(new KeyOnlyFilter()); + } else { + boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); + for (int eachIndex : targetIndexes) { + if (isRowKeyMappings[eachIndex]) { + continue; + } + byte[][] mappingColumn = columnMapping.getMappingColumns()[eachIndex]; + if (mappingColumn[1] == null) { + scan.addFamily(mappingColumn[0]); + } else { + scan.addColumn(mappingColumn[0], mappingColumn[1]); + } + } + } + scan.setStartRow(fragment.getStartRow()); if (fragment.isLast() && fragment.getStopRow() != null && fragment.getStopRow().length > 0) { // last and stopRow is not empty - Filter filter = new InclusiveStopFilter(fragment.getStopRow()); - scan.setFilter(filter); + if (filters == null) { + filters = new FilterList(); + } + filters.addFilter(new InclusiveStopFilter(fragment.getStopRow())); } else { scan.setStopRow(fragment.getStopRow()); } - boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); - for (int eachIndex: targetIndexes) { - if (isRowKeyMappings[eachIndex]) { - continue; - } - byte[][] mappingColumn = columnMapping.getMappingColumns()[eachIndex]; - if (mappingColumn[1] == null) { - scan.addFamily(mappingColumn[0]); - } else { - scan.addColumn(mappingColumn[0], mappingColumn[1]); - } + if (filters != null) { + scan.setFilter(filters); } if (htable == null) { diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java index 77967743a3..e08bfca0f9 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java @@ -34,6 +34,14 @@ public class TestColumnMapping { @Test public void testColumnKeyValueMapping() throws Exception { + if (1 == 1) { + long interval = 600000000L/50; + + for (int i = 0; i < 50; i++) { + System.out.println(interval * (i + 1)); + } + return; + } KeyValueSet keyValueSet = new KeyValueSet(); keyValueSet.set(HBaseStorageManager.META_TABLE_KEY, "test"); keyValueSet.set(HBaseStorageManager.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:"); From 12b30c15fb038b757fbf53c393d79336b7232514 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Wed, 5 Nov 2014 18:07:00 +0900 Subject: [PATCH 23/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Add comments --- .../apache/tajo/master/querymaster/Query.java | 4 - .../master/querymaster/Repartitioner.java | 2 - .../tajo/master/querymaster/SubQuery.java | 3 - .../java/org/apache/tajo/worker/Task.java | 1 - .../tajo/storage/FileStorageManager.java | 3 - .../storage/HashShuffleAppenderManager.java | 1 - .../apache/tajo/storage/StorageManager.java | 386 ++++++++++++++---- .../storage/hbase/AbstractHBaseAppender.java | 7 +- .../hbase/AddSortForInsertRewriter.java | 1 - .../HBaseBinarySerializerDeserializer.java | 2 - .../tajo/storage/hbase/HBasePutAppender.java | 1 - .../tajo/storage/hbase/HBaseScanner.java | 1 - .../storage/hbase/HBaseStorageManager.java | 36 ++ .../HBaseTextSerializerDeserializer.java | 2 - .../tajo/storage/hbase/HFileAppender.java | 8 - .../tajo/storage/hbase/TestColumnMapping.java | 10 +- 16 files changed, 352 insertions(+), 116 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index c15c4cb498..ac928c1dae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -19,13 +19,10 @@ package org.apache.tajo.master.querymaster; import com.google.common.collect.Maps; -import org.apache.avro.ipc.trace.TracePlugin.StorageType; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.event.EventHandler; @@ -36,7 +33,6 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index f558433a0d..b5119fdfb2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -23,7 +23,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; -import org.apache.tajo.ConfigKey; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; @@ -50,7 +49,6 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.RowStoreUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 17c1332d29..067d07bc52 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -23,7 +23,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; @@ -37,7 +36,6 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.StatisticsUtil; @@ -63,7 +61,6 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 71e6522fd8..61d4989788 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryUnitAttemptId; -import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoProtos; import org.apache.tajo.TajoProtos.TaskAttemptState; import org.apache.tajo.catalog.Schema; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index 019d1930a4..d382f41fc9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -32,11 +32,8 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.*; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index ece31dd9b5..33a9233dfd 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -31,7 +31,6 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.util.Pair; -import org.apache.tajo.storage.StorageManager; import java.io.IOException; import java.util.ArrayList; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 8b423bc2c2..1e81816b29 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -47,14 +47,36 @@ import java.util.concurrent.ConcurrentHashMap; /** - * StorageManager + * StorageManager manages the functions of storing and reading data. + * StorageManager is a abstract class. + * For supporting such as HDFS, HBASE, a specific StorageManager should be implemented by inheriting this class. + * */ public abstract class StorageManager { private final Log LOG = LogFactory.getLog(StorageManager.class); + private static final Class[] DEFAULT_SCANNER_PARAMS = { + Configuration.class, + Schema.class, + TableMeta.class, + Fragment.class + }; + + private static final Class[] DEFAULT_APPENDER_PARAMS = { + Configuration.class, + QueryUnitAttemptId.class, + Schema.class, + TableMeta.class, + Path.class + }; + protected TajoConf conf; protected StoreType storeType; + /** + * Cache of StorageManager. + * Key is manager key(warehouse path) + store type + */ private static final Map storageManagers = Maps.newHashMap(); /** @@ -79,21 +101,26 @@ public abstract class StorageManager { public StorageManager(StoreType storeType) { this.storeType = storeType; } + /** - * + * Initialize storage manager. * @throws IOException */ protected abstract void storageInit() throws IOException; /** + * This method is called after executing "CREATE TABLE" statement. + * If a storage is a file based storage, a storage manager may create directory. * - * @param tableDesc - * @param ifNotExists + * @param tableDesc Table description which is created. + * @param ifNotExists Creates the table only when the table does not exist. * @throws IOException */ public abstract void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException; /** + * This method is called after executing "DROP TABLE" statement with the 'PURGE' option + * which is the option to delete all the data. * * @param tableDesc * @throws IOException @@ -101,29 +128,32 @@ public StorageManager(StoreType storeType) { public abstract void purgeTable(TableDesc tableDesc) throws IOException; /** - * - * @param fragmentId - * @param tableDesc - * @param scanNode - * @return + * Returns the splits that will serve as input for the scan tasks. The + * number of splits matches the number of regions in a table. + * @param fragmentId The table name or previous ExecutionBlockId + * @param tableDesc The table description for the target data. + * @param scanNode The logical node for scanning. + * @return The list of input fragments. * @throws IOException */ public abstract List getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException; /** - * - * @param tableDesc - * @param currentPage - * @param numFragments - * @return + * It returns the splits that will serve as input for the non-forward query scanner such as 'select * from table1'. + * The result list should be small. If there is many fragments for scanning, TajoMaster uses the paging navigation. + * @param tableDesc The table description for the target data. + * @param currentPage The current page number within the entire list. + * @param numFragments The number of fragments in the result. + * @return The list of input fragments. * @throws IOException */ public abstract List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException; /** - * @return + * It returns the storage property. + * @return The storage property */ public abstract StorageProperty getStorageProperty(); @@ -133,12 +163,15 @@ public abstract List getNonForwardSplit(TableDesc tableDesc, int curre public abstract void closeStorageManager(); /** + * It is called by a Repartitioner for range shuffling when the SortRangeType of SortNode is USING_STORAGE_MANAGER. + * In general Repartitioner determines the partition range using previous output statistics data. + * In the special cases, such as HBase Repartitioner uses the result of this method. * - * @param queryContext - * @param tableDesc - * @param inputSchema - * @param sortSpecs - * @return + * @param queryContext The current query context which contains query properties. + * @param tableDesc The table description for the target data. + * @param inputSchema The input schema + * @param sortSpecs The sort specification that contains the sort column and sort order. + * @return The list of sort ranges. * @throws IOException */ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, @@ -146,27 +179,48 @@ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, T TupleRange dataRange) throws IOException; /** - * @param node + * This method is called before executing 'INSERT' or 'CREATE TABLE as SELECT'. + * In general Tajo creates the target table after finishing the final sub-query of CATS. + * But In the special cases, such as HBase INSERT or CAST query uses the target table information. + * That kind of the storage should implements the logic related to creating table in this method. + * + * @param node The child node of the root node. * @throws IOException */ public abstract void beforeInsertOrCATS(LogicalNode node) throws IOException; /** + * It is called when the query failed. + * Each storage manager should implement to be processed when the query fails in this method. * - * @param node + * @param node The child node of the root node. * @throws IOException */ public abstract void queryFailed(LogicalNode node) throws IOException; + /** + * Returns the current storage type. + * @return + */ public StoreType getStoreType() { return storeType; } + /** + * Initialize StorageManager instance. It should be called before using. + * + * @param tajoConf + * @throws IOException + */ public void init(TajoConf tajoConf) throws IOException { this.conf = tajoConf; storageInit(); } + /** + * Close StorageManager + * @throws IOException + */ public void close() throws IOException { synchronized(storageManagers) { for (StorageManager eachStorageManager: storageManagers.values()) { @@ -175,14 +229,38 @@ public void close() throws IOException { } } + /** + * Returns the splits that will serve as input for the scan tasks. The + * number of splits matches the number of regions in a table. + * + * @param fragmentId The table name or previous ExecutionBlockId + * @param tableDesc The table description for the target data. + * @return The list of input fragments. + * @throws IOException + */ public List getSplits(String fragmentId, TableDesc tableDesc) throws IOException { return getSplits(fragmentId, tableDesc, null); } + /** + * Returns FileStorageManager instance. + * + * @param tajoConf Tajo system property. + * @return + * @throws IOException + */ public static FileStorageManager getFileStorageManager(TajoConf tajoConf) throws IOException { return getFileStorageManager(tajoConf, null); } + /** + * Returns FileStorageManager instance and sets WAREHOUSE_DIR property in tajoConf with warehousePath parameter. + * + * @param tajoConf Tajo system property. + * @param warehousePath The warehouse directory to be set in the tajoConf. + * @return + * @throws IOException + */ public static FileStorageManager getFileStorageManager(TajoConf tajoConf, Path warehousePath) throws IOException { URI uri; TajoConf copiedConf = new TajoConf(tajoConf); @@ -194,6 +272,14 @@ public static FileStorageManager getFileStorageManager(TajoConf tajoConf, Path w return (FileStorageManager) getStorageManager(copiedConf, StoreType.CSV, key); } + /** + * Returns the proper StorageManager instance according to the storeType. + * + * @param tajoConf Tajo system property. + * @param storeType Storage type + * @return + * @throws IOException + */ public static StorageManager getStorageManager(TajoConf tajoConf, String storeType) throws IOException { if ("HBASE".equals(storeType)) { return getStorageManager(tajoConf, StoreType.HBASE); @@ -202,12 +288,29 @@ public static StorageManager getStorageManager(TajoConf tajoConf, String storeTy } } + /** + * Returns the proper StorageManager instance according to the storeType. + * + * @param tajoConf Tajo system property. + * @param storeType Storage type + * @return + * @throws IOException + */ public static StorageManager getStorageManager(TajoConf tajoConf, StoreType storeType) throws IOException { return getStorageManager(tajoConf, storeType, null); } + /** + * Returns the proper StorageManager instance according to the storeType + * + * @param tajoConf Tajo system property. + * @param storeType Storage type + * @param managerKey Key that can identify each storage manager(may be a path) + * @return + * @throws IOException + */ public static synchronized StorageManager getStorageManager ( - TajoConf conf, StoreType storeType, String managerKey) throws IOException { + TajoConf tajoConf, StoreType storeType, String managerKey) throws IOException { synchronized (storageManagers) { String storeKey = storeType + managerKey; StorageManager manager = storageManagers.get(storeKey); @@ -220,7 +323,7 @@ public static synchronized StorageManager getStorageManager ( manager = new FileStorageManager(storeType); } - manager.init(conf); + manager.init(tajoConf); storageManagers.put(storeKey, manager); } @@ -228,14 +331,108 @@ public static synchronized StorageManager getStorageManager ( } } + /** + * Returns Scanner instance. + * + * @param meta The table meta + * @param schema The input schema + * @param fragment The fragment for scanning + * @param target Columns which are selected. + * @return Scanner instance + * @throws IOException + */ public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException { return getScanner(meta, schema, FragmentConvertor.convert(conf, fragment), target); } + /** + * Returns Scanner instance. + * + * @param meta The table meta + * @param schema The input schema + * @param fragment The fragment for scanning + * @return Scanner instance + * @throws IOException + */ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException { return getScanner(meta, schema, fragment, schema); } + /** + * Returns Scanner instance. + * + * @param meta The table meta + * @param schema The input schema + * @param fragment The fragment for scanning + * @param target The output schema + * @return Scanner instance + * @throws IOException + */ + public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { + if (fragment.isEmpty()) { + Scanner scanner = new NullScanner(conf, schema, meta, fragment); + scanner.setTarget(target.toArray()); + + return scanner; + } + + Scanner scanner; + + Class scannerClass = getScannerClass(meta.getStoreType()); + scanner = newScannerInstance(scannerClass, conf, schema, meta, fragment); + if (scanner.isProjectable()) { + scanner.setTarget(target.toArray()); + } + + return scanner; + } + + /** + * Returns Scanner instance. + * + * @param conf The system property + * @param meta The table meta + * @param schema The input schema + * @param fragment The fragment for scanning + * @param target The output schema + * @return Scanner instance + * @throws IOException + */ + public static synchronized SeekableScanner getSeekableScanner( + TajoConf conf, TableMeta meta, Schema schema, FileFragment fragment, Schema target) throws IOException { + return (SeekableScanner)getStorageManager(conf, meta.getStoreType()).getScanner(meta, schema, fragment, target); + } + + /** + * Returns Scanner instance. + * + * @param conf The system property + * @param meta The table meta + * @param schema The input schema + * @param path The data file path + * @return Scanner instance + * @throws IOException + */ + public static synchronized SeekableScanner getSeekableScanner( + TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { + + FileSystem fs = path.getFileSystem(conf); + FileStatus status = fs.getFileStatus(path); + FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); + + return getSeekableScanner(conf, meta, schema, fragment, schema); + } + + /** + * Returns Appender instance. + * @param queryContext Query property. + * @param taskAttemptId Task id. + * @param meta Table meta data. + * @param schema Output schema. + * @param workDir Working directory + * @return Appender instance + * @throws IOException + */ public Appender getAppender(OverridableConf queryContext, QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) throws IOException { @@ -261,23 +458,16 @@ public Appender getAppender(OverridableConf queryContext, return appender; } - private static final Class[] DEFAULT_SCANNER_PARAMS = { - Configuration.class, - Schema.class, - TableMeta.class, - Fragment.class - }; - - private static final Class[] DEFAULT_APPENDER_PARAMS = { - Configuration.class, - QueryUnitAttemptId.class, - Schema.class, - TableMeta.class, - Path.class - }; - /** - * create a scanner instance. + * Creates a scanner instance. + * + * @param theClass Concrete class of scanner + * @param conf System property + * @param schema Input schema + * @param meta Table meta data + * @param fragment The fragment for scanning + * @param + * @return The scanner instance */ public static T newScannerInstance(Class theClass, Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { @@ -298,7 +488,16 @@ public static T newScannerInstance(Class theClass, Configuration conf, Sc } /** - * create a scanner instance. + * Creates a scanner instance. + * + * @param theClass Concrete class of scanner + * @param conf System property + * @param taskAttemptId Task id + * @param meta Table meta data + * @param schema Input schema + * @param workDir Working directory + * @param + * @return The scanner instance */ public static T newAppenderInstance(Class theClass, Configuration conf, QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) { @@ -318,6 +517,13 @@ public static T newAppenderInstance(Class theClass, Configuration conf, Q return result; } + /** + * Return the Scanner class for the StoreType that is defined in storage-default.xml. + * + * @param storeType store type + * @return The Scanner class + * @throws IOException + */ public Class getScannerClass(CatalogProtos.StoreType storeType) throws IOException { String handlerName = storeType.name().toLowerCase(); Class scannerClass = SCANNER_HANDLER_CACHE.get(handlerName); @@ -334,40 +540,14 @@ public Class getScannerClass(CatalogProtos.StoreType storeTyp return scannerClass; } - public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { - if (fragment.isEmpty()) { - Scanner scanner = new NullScanner(conf, schema, meta, fragment); - scanner.setTarget(target.toArray()); - - return scanner; - } - - Scanner scanner; - - Class scannerClass = getScannerClass(meta.getStoreType()); - scanner = newScannerInstance(scannerClass, conf, schema, meta, fragment); - if (scanner.isProjectable()) { - scanner.setTarget(target.toArray()); - } - - return scanner; - } - - public static synchronized SeekableScanner getSeekableScanner( - TajoConf conf, TableMeta meta, Schema schema, FileFragment fragment, Schema target) throws IOException { - return (SeekableScanner)getStorageManager(conf, meta.getStoreType()).getScanner(meta, schema, fragment, target); - } - - public static synchronized SeekableScanner getSeekableScanner( - TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { - - FileSystem fs = path.getFileSystem(conf); - FileStatus status = fs.getFileStatus(path); - FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); - - return getSeekableScanner(conf, meta, schema, fragment, schema); - } - + /** + * Return length of the fragment. + * In the UNKNOWN_LENGTH case get FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH from the configuration. + * + * @param conf Tajo system property + * @param fragment Fragment + * @return + */ public static long getFragmentLength(TajoConf conf, Fragment fragment) { if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { return conf.getLongVar(ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); @@ -376,20 +556,63 @@ public static long getFragmentLength(TajoConf conf, Fragment fragment) { } } + /** + * It is called after making logical plan. Storage manager should verify the schema for inserting. + * + * @param tableDesc The table description of insert target. + * @param outSchema The output schema of select query for inserting. + * @throws IOException + */ public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { // nothing to do } + /** + * Returns the list of storage specified rewrite rules. + * This values are used by LogicalOptimizer. + * + * @param queryContext The query property + * @param tableDesc The description of the target table. + * @return The list of storage specified rewrite rules + * @throws IOException + */ public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { return null; } + /** + * Finalizes result data. Tajo stores result data in the staging directory. + * If the query fails, clean up the staging directory. + * Otherwise the query is successful, move to the final directory from the staging directory. + * + * @param queryContext The query property + * @param finalEbId The final execution block id + * @param plan The query plan + * @param schema The final output schema + * @param tableDesc The description of the target table + * @return Saved path + * @throws IOException + */ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { return commitOutputData(queryContext, finalEbId, plan, schema, tableDesc, true); } + /** + * Finalizes result data. Tajo stores result data in the staging directory. + * If the query fails, clean up the staging directory. + * Otherwise the query is successful, move to the final directory from the staging directory. + * + * @param queryContext The query property + * @param finalEbId The final execution block id + * @param plan The query plan + * @param schema The final output schema + * @param tableDesc The description of the target table + * @param changeFileSeq If true change result file name with max sequence. + * @return Saved path + * @throws IOException + */ protected Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc, boolean changeFileSeq) throws IOException { @@ -576,6 +799,12 @@ private void moveResultFromStageToFinal(FileSystem fs, Path stagingResultDir, } } + /** + * Removes the path of the parent. + * @param parentPath + * @param childPath + * @return + */ private String extractSubPath(Path parentPath, Path childPath) { String parentPathStr = parentPath.toUri().getPath(); String childPathStr = childPath.toUri().getPath(); @@ -609,6 +838,13 @@ private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IO return tokens[0] + "-" + tokens[1] + "-" + tokens[2] + "-" + nf.format(seq); } + /** + * Make sure all files are moved. + * @param fs FileSystem + * @param stagingPath The stagind directory + * @return + * @throws IOException + */ private boolean verifyAllFileMoved(FileSystem fs, Path stagingPath) throws IOException { FileStatus[] files = fs.listStatus(stagingPath); if (files != null && files.length != 0) { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java index f5e9b9ab27..861523590b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java @@ -21,16 +21,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HConnection; import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TableStatistics; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.TUtil; @@ -42,6 +38,9 @@ import java.util.List; import java.util.Map; +/** + * An abstract class for HBase appender. + */ public abstract class AbstractHBaseAppender implements Appender { protected Configuration conf; protected Schema schema; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java index e23c44df6f..8aefebba73 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java @@ -18,7 +18,6 @@ package org.apache.tajo.storage.hbase; -import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java index 3bfeea8c0f..c05c5bb0d5 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java @@ -22,11 +22,9 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.storage.SerializerDeserializer; import org.apache.tajo.util.Bytes; import java.io.IOException; -import java.io.OutputStream; public class HBaseBinarySerializerDeserializer { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java index 674e002212..50f61a8a4a 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java @@ -20,7 +20,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index 04e7caebe0..c011eadda8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -38,7 +38,6 @@ import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.util.Bytes; import org.apache.tajo.util.BytesUtils; import java.io.IOException; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 3a86520aa5..142e7d3e9b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -53,6 +53,9 @@ import java.io.InputStreamReader; import java.util.*; +/** + * StorageManager for HBase table. + */ public class HBaseStorageManager extends StorageManager { private static final Log LOG = LogFactory.getLog(HBaseStorageManager.class); @@ -186,6 +189,15 @@ private void createTable(TableMeta tableMeta, Schema schema, } } + /** + * Returns initial region split keys. + * + * @param conf + * @param schema + * @param meta + * @return + * @throws IOException + */ private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) throws IOException { String splitRowKeys = meta.getOption(META_SPLIT_ROW_KEYS_KEY, ""); String splitRowKeysFile = meta.getOption(META_SPLIT_ROW_KEYS_FILE_KEY, ""); @@ -231,6 +243,7 @@ private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) thro } if (splitRowKeysFile != null && !splitRowKeysFile.isEmpty()) { + // If there is many split keys, Tajo allows to define in the file. Path path = new Path(splitRowKeysFile); FileSystem fs = path.getFileSystem(conf); if (!fs.exists(path)) { @@ -274,6 +287,14 @@ private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) thro return null; } + /** + * Creates Configuration instance and sets with hbase connection options. + * + * @param conf + * @param tableMeta + * @return + * @throws IOException + */ public static Configuration getHBaseConfiguration(Configuration conf, TableMeta tableMeta) throws IOException { String zkQuorum = tableMeta.getOption(META_ZK_QUORUM_KEY, ""); if (zkQuorum == null || zkQuorum.trim().isEmpty()) { @@ -292,6 +313,14 @@ public static Configuration getHBaseConfiguration(Configuration conf, TableMeta return hbaseConf; } + /** + * Creates HTableDescription using table meta data. + * + * @param tableMeta + * @param schema + * @return + * @throws IOException + */ public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema schema) throws IOException { String hbaseTableName = tableMeta.getOption(META_TABLE_KEY, ""); if (hbaseTableName == null || hbaseTableName.trim().isEmpty()) { @@ -332,6 +361,13 @@ public void purgeTable(TableDesc tableDesc) throws IOException { } } + /** + * Returns columns which are mapped to the rowkey of the hbase table. + * + * @param tableDesc + * @return + * @throws IOException + */ private Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java index 52b76f6504..a0ad49235c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java @@ -22,11 +22,9 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.storage.SerializerDeserializer; import org.apache.tajo.util.NumberUtil; import java.io.IOException; -import java.io.OutputStream; public class HBaseTextSerializerDeserializer { public static Datum deserialize(Column col, byte[] bytes) throws IOException { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java index e2fce2b594..b9425f9fad 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java @@ -35,22 +35,14 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryUnitAttemptId; -import org.apache.tajo.QueryVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.Datum; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.TableStatistics; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.Bytes; -import org.apache.tajo.util.TUtil; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.TreeSet; public class HFileAppender extends AbstractHBaseAppender { diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java index e08bfca0f9..f242b118bf 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java @@ -18,10 +18,12 @@ package org.apache.tajo.storage.hbase; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.util.Bytes; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; @@ -34,14 +36,6 @@ public class TestColumnMapping { @Test public void testColumnKeyValueMapping() throws Exception { - if (1 == 1) { - long interval = 600000000L/50; - - for (int i = 0; i < 50; i++) { - System.out.println(interval * (i + 1)); - } - return; - } KeyValueSet keyValueSet = new KeyValueSet(); keyValueSet.set(HBaseStorageManager.META_TABLE_KEY, "test"); keyValueSet.set(HBaseStorageManager.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:"); From 44e4d632df99c340ad520a1c39ba3efff94ebde7 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Wed, 5 Nov 2014 20:27:21 +0900 Subject: [PATCH 24/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Change TupleComparator to BaseTupleComparator. --- .../src/main/java/org/apache/tajo/plan/util/PlannerUtil.java | 4 +--- .../org/apache/tajo/storage/hbase/HBaseStorageManager.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 732e4357ee..a7db188bdd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -23,9 +23,7 @@ import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.DataType; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 142e7d3e9b..5cdbe444e4 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -988,7 +988,7 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc } List tupleRanges = new ArrayList(endKeys.length); - TupleComparator comparator = new TupleComparator(inputSchema, sortSpecs); + TupleComparator comparator = new BaseTupleComparator(inputSchema, sortSpecs); Tuple previousTuple = dataRange.getStart(); for (byte[] eachEndKey : endKeys) { From 03ef3158930624acc44ce482f10abcbdcc01e9b4 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Mon, 10 Nov 2014 18:58:05 +0900 Subject: [PATCH 25/29] TAJO-1131: Supports Inserting or Creating table into the HBase mapped table. Move hbase.version property to tajo-project/pom.xml --- BUILDING | 2 ++ tajo-core/pom.xml | 1 - tajo-dist/src/main/bin/tajo | 15 +++++++++++++++ tajo-dist/src/main/conf/tajo-env.sh | 3 +++ tajo-project/pom.xml | 1 + tajo-storage/pom.xml | 3 ++- 6 files changed, 23 insertions(+), 2 deletions(-) diff --git a/BUILDING b/BUILDING index 3be765ed8b..9b1bda58e0 100644 --- a/BUILDING +++ b/BUILDING @@ -44,6 +44,8 @@ Maven build goals: * Use -Dtar to create a TAR with the distribution (using -Pdist) * Use -Dhadoop.version to build with the specific hadoop version (-Dhadoop.version=2.5.1) * Currently, 2.2.0 or higher are supported. + * Use -Dhbase.version to build with the specific hbase version (-Dhbase.version=0.98.7-hadoop2) + * Currently, 0.98.x-hadoop2 or higher are tested. Tests options: * Use -DskipTests to skip tests when running the following Maven goals: diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index 37a59459cf..b58ae89d29 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -34,7 +34,6 @@ true true 3.0.1 - 0.98.6.1-hadoop2 diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo index d4e99f9d1d..8a9c17a2c9 100755 --- a/tajo-dist/src/main/bin/tajo +++ b/tajo-dist/src/main/bin/tajo @@ -225,6 +225,21 @@ HDFS_LIBRARY_PATH="${HADOOP_HOME}/lib/native/" # Hadoop Home Configuration End ############################################################################## +############################################################################## +# Set HBase CLASSPATH +############################################################################## +if [ "$HBASE_HOME" != "" ]; then + for f in ${HBASE_HOME}/lib/hbase-*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done + for f in ${HBASE_HOME}/lib/htrace-*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done +fi +############################################################################## +# HBase Configuration End +############################################################################## + ############################################################################## # Find and Set Hive CLASSPATH ############################################################################## diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh index 92d00bd8bb..064a82f0a2 100755 --- a/tajo-dist/src/main/conf/tajo-env.sh +++ b/tajo-dist/src/main/conf/tajo-env.sh @@ -24,6 +24,9 @@ # Hadoop home. Required # export HADOOP_HOME= +# HBase home. optional +# export HBASE_HOME= + # The java implementation to use. Required. # export JAVA_HOME=/usr/java/default diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 97489daa55..df8e814bf5 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -36,6 +36,7 @@ 2.5.1 2.5.0 0.9.1-SNAPSHOT + 0.98.7-hadoop2 ${project.parent.relativePath}/.. src/main/hadoop-${hadoop.version} diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index c0a835a2ba..5a265e8ce9 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -34,7 +34,6 @@ UTF-8 1.5.0 2.1.0 - 0.98.6.1-hadoop2 @@ -330,11 +329,13 @@ org.apache.hbase hbase-server ${hbase.version} + provided org.apache.hbase hbase-client ${hbase.version} + provided From 445019d32c5d3f10522fc11fe6a5ee939f389954 Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Tue, 11 Nov 2014 19:19:28 +0900 Subject: [PATCH 26/29] TAJO-1122: Refactor the tajo-storage project structure. --- .../src/main/proto/CatalogProtos.proto | 9 - tajo-client/pom.xml | 6 +- .../org/apache/tajo/jdbc/TajoResultSet.java | 5 +- tajo-core/pom.xml | 12 +- .../engine/planner/PhysicalPlannerImpl.java | 14 +- .../physical/ColPartitionStoreExec.java | 3 +- .../planner/physical/PhysicalPlanUtil.java | 6 +- .../physical/RangeShuffleFileWriteExec.java | 4 +- .../planner/physical/StoreTableExec.java | 4 +- .../apache/tajo/master/LazyTaskScheduler.java | 3 +- .../org/apache/tajo/master/TajoMaster.java | 2 +- .../master/querymaster/QueryMasterTask.java | 1 - .../master/querymaster/Repartitioner.java | 14 +- .../tajo/master/querymaster/SubQuery.java | 2 +- .../java/org/apache/tajo/worker/Task.java | 10 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../tajo/engine/planner/TestPlannerUtil.java | 2 +- .../planner/global/TestBroadcastJoinPlan.java | 9 +- .../planner/physical/TestBNLJoinExec.java | 6 +- .../planner/physical/TestBSTIndexExec.java | 5 +- .../physical/TestExternalSortExec.java | 5 +- .../physical/TestFullOuterHashJoinExec.java | 15 +- .../physical/TestFullOuterMergeJoinExec.java | 16 +- .../physical/TestHashAntiJoinExec.java | 7 +- .../planner/physical/TestHashJoinExec.java | 7 +- .../physical/TestHashSemiJoinExec.java | 7 +- .../physical/TestLeftOuterHashJoinExec.java | 13 +- .../physical/TestLeftOuterNLJoinExec.java | 13 +- .../planner/physical/TestMergeJoinExec.java | 7 +- .../planner/physical/TestNLJoinExec.java | 6 +- .../planner/physical/TestPhysicalPlanner.java | 21 +- .../TestProgressExternalSortExec.java | 3 +- .../physical/TestRightOuterHashJoinExec.java | 9 +- .../physical/TestRightOuterMergeJoinExec.java | 16 +- .../engine/planner/physical/TestSortExec.java | 5 +- .../tajo/engine/query/TestJoinBroadcast.java | 4 +- .../org/apache/tajo/jdbc/TestResultSet.java | 5 +- .../org/apache/tajo/storage/TestRowFile.java | 6 +- .../worker/TestRangeRetrieverHandler.java | 4 +- tajo-dist/pom.xml | 7 +- tajo-jdbc/pom.xml | 6 +- tajo-project/pom.xml | 16 + tajo-storage/pom.xml | 327 +--------- .../tajo/tuple/TestBaseTupleBuilder.java | 76 --- .../tajo/tuple/offheap/TestHeapTuple.java | 45 -- .../tuple/offheap/TestOffHeapRowBlock.java | 577 ------------------ .../tajo/tuple/offheap/TestResizableSpec.java | 59 -- tajo-storage/tajo-storage-common/pom.xml | 333 ++++++++++ .../org/apache/tajo/storage/Appender.java | 0 .../tajo/storage/BaseTupleComparator.java | 0 .../storage/BinarySerializerDeserializer.java | 0 .../org/apache/tajo/storage/DataLocation.java | 0 .../apache/tajo/storage/DiskDeviceInfo.java | 0 .../org/apache/tajo/storage/DiskInfo.java | 0 .../apache/tajo/storage/DiskMountInfo.java | 0 .../org/apache/tajo/storage/DiskUtil.java | 0 .../org/apache/tajo/storage/FrameTuple.java | 0 .../org/apache/tajo/storage/LazyTuple.java | 0 .../org/apache/tajo/storage/MemoryUtil.java | 0 .../org/apache/tajo/storage/MergeScanner.java | 0 .../org/apache/tajo/storage/NullScanner.java | 55 +- .../tajo/storage/NumericPathComparator.java | 0 .../org/apache/tajo/storage/RowStoreUtil.java | 0 .../java/org/apache/tajo/storage/Scanner.java | 18 +- .../apache/tajo/storage/SeekableScanner.java | 0 .../tajo/storage/SerializerDeserializer.java | 0 .../apache/tajo/storage/StorageManager.java | 132 ++-- .../apache/tajo/storage/StorageProperty.java | 0 .../org/apache/tajo/storage/StorageUtil.java | 27 - .../apache/tajo/storage/TableStatistics.java | 0 .../storage/TextSerializerDeserializer.java | 0 .../apache/tajo/storage/TupleComparator.java | 0 .../org/apache/tajo/storage/TupleRange.java | 0 .../storage/annotation/ForSplitableStore.java | 0 .../tajo/storage/compress/CodecPool.java | 0 .../AlreadyExistsStorageException.java | 0 .../exception/UnknownCodecException.java | 0 .../exception/UnknownDataTypeException.java | 0 .../UnsupportedFileTypeException.java | 0 .../tajo/storage/fragment/Fragment.java | 0 .../storage/fragment/FragmentConvertor.java | 0 .../apache/tajo/tuple/BaseTupleBuilder.java | 0 .../org/apache/tajo/tuple/RowBlockReader.java | 0 .../org/apache/tajo/tuple/TupleBuilder.java | 0 .../tajo/tuple/offheap/DirectBufTuple.java | 0 .../tuple/offheap/FixedSizeLimitSpec.java | 0 .../apache/tajo/tuple/offheap/HeapTuple.java | 0 .../tajo/tuple/offheap/OffHeapMemory.java | 0 .../tajo/tuple/offheap/OffHeapRowBlock.java | 0 .../tuple/offheap/OffHeapRowBlockReader.java | 0 .../tuple/offheap/OffHeapRowBlockUtils.java | 0 .../tuple/offheap/OffHeapRowBlockWriter.java | 0 .../tajo/tuple/offheap/OffHeapRowWriter.java | 0 .../tuple/offheap/ResizableLimitSpec.java | 0 .../apache/tajo/tuple/offheap/RowWriter.java | 0 .../tajo/tuple/offheap/UnSafeTuple.java | 0 .../offheap/UnSafeTupleBytesComparator.java | 0 .../tajo/tuple/offheap/ZeroCopyTuple.java | 0 .../src/main/proto/IndexProtos.proto | 0 .../src/main/resources/storage-default.xml | 10 + .../apache/tajo/storage/TestFrameTuple.java | 4 +- .../apache/tajo/storage/TestLazyTuple.java | 0 .../tajo/storage/TestTupleComparator.java | 0 .../org/apache/tajo/storage/TestVTuple.java | 4 +- .../src/test/resources/storage-default.xml | 164 +++++ tajo-storage/tajo-storage-hbase/pom.xml | 349 +++++++++++ .../storage/hbase/AbstractHBaseAppender.java | 0 .../hbase/AddSortForInsertRewriter.java | 5 +- .../tajo/storage/hbase/ColumnMapping.java | 0 .../HBaseBinarySerializerDeserializer.java | 0 .../tajo/storage/hbase/HBaseFragment.java | 2 +- .../tajo/storage/hbase/HBasePutAppender.java | 0 .../tajo/storage/hbase/HBaseScanner.java | 5 +- .../storage/hbase/HBaseStorageManager.java | 23 +- .../HBaseTextSerializerDeserializer.java | 0 .../tajo/storage/hbase/HFileAppender.java | 2 +- .../tajo/storage/hbase/IndexPredication.java | 0 .../tajo/storage/hbase/RowKeyMapping.java | 0 .../main/proto/StorageFragmentProtos.proto | 2 +- .../tajo/storage/hbase/TestColumnMapping.java | 6 +- .../hbase/TestHBaseStorageManager.java | 0 tajo-storage/tajo-storage-hdfs/pom.xml | 398 ++++++++++++ .../java/org/apache/tajo/storage/CSVFile.java | 1 - .../storage/CompressedSplitLineReader.java | 0 .../org/apache/tajo/storage/FileAppender.java | 3 +- .../org/apache/tajo/storage/FileScanner.java | 0 .../tajo/storage/FileStorageManager.java | 32 +- .../tajo/storage/HashShuffleAppender.java | 2 +- .../storage/HashShuffleAppenderManager.java | 4 +- .../org/apache/tajo/storage/LineReader.java | 18 +- .../java/org/apache/tajo/storage/RawFile.java | 1 - .../java/org/apache/tajo/storage/RowFile.java | 2 +- .../apache/tajo/storage/SplitLineReader.java | 0 .../tajo/storage/avro/AvroAppender.java | 14 +- .../apache/tajo/storage/avro/AvroScanner.java | 1 - .../apache/tajo/storage/avro/AvroUtil.java | 6 +- .../tajo/storage/avro/package-info.java | 0 .../tajo/storage/fragment/FileFragment.java | 2 +- .../tajo/storage/index/IndexMethod.java | 5 +- .../tajo/storage/index/IndexReader.java | 2 +- .../tajo/storage/index/IndexWriter.java | 0 .../tajo/storage/index/OrderIndexReader.java | 6 +- .../tajo/storage/index/bst/BSTIndex.java | 10 +- .../tajo/storage/parquet/ParquetAppender.java | 9 +- .../tajo/storage/parquet/ParquetScanner.java | 0 .../storage/parquet/TajoParquetReader.java | 0 .../storage/parquet/TajoParquetWriter.java | 6 +- .../tajo/storage/parquet/TajoReadSupport.java | 14 +- .../storage/parquet/TajoRecordConverter.java | 30 +- .../parquet/TajoRecordMaterializer.java | 7 +- .../storage/parquet/TajoSchemaConverter.java | 22 +- .../storage/parquet/TajoWriteSupport.java | 23 +- .../tajo/storage/parquet/package-info.java | 0 .../storage/rcfile/BytesRefArrayWritable.java | 0 .../tajo/storage/rcfile/BytesRefWritable.java | 8 +- .../storage/rcfile/ColumnProjectionUtils.java | 0 .../rcfile/LazyDecompressionCallback.java | 2 +- .../rcfile/NonSyncByteArrayInputStream.java | 0 .../rcfile/NonSyncByteArrayOutputStream.java | 0 .../rcfile/NonSyncDataInputBuffer.java | 0 .../rcfile/NonSyncDataOutputBuffer.java | 0 .../apache/tajo/storage/rcfile/RCFile.java | 33 +- .../SchemaAwareCompressionInputStream.java | 0 .../SchemaAwareCompressionOutputStream.java | 0 .../sequencefile/SequenceFileAppender.java | 0 .../sequencefile/SequenceFileScanner.java | 0 .../thirdparty/parquet/CodecFactory.java | 20 +- .../parquet/ColumnChunkPageWriteStore.java | 19 +- .../parquet/InternalParquetRecordReader.java | 7 +- .../parquet/InternalParquetRecordWriter.java | 20 +- .../thirdparty/parquet/ParquetFileWriter.java | 50 +- .../thirdparty/parquet/ParquetReader.java | 27 +- .../thirdparty/parquet/ParquetWriter.java | 22 +- .../tajo/storage/trevni/TrevniAppender.java | 0 .../tajo/storage/trevni/TrevniScanner.java | 0 .../main/proto/StorageFragmentProtos.proto} | 39 +- .../tajo/storage/TestCompressionStorages.java | 4 +- .../tajo/storage/TestFileStorageManager.java} | 12 +- .../apache/tajo/storage/TestFileSystems.java | 3 +- .../apache/tajo/storage/TestMergeScanner.java | 8 +- .../org/apache/tajo/storage/TestStorages.java | 46 +- .../tajo/storage/index/TestBSTIndex.java | 51 +- .../index/TestSingleCSVFileBSTIndex.java | 4 +- .../tajo/storage/parquet/TestReadWrite.java | 17 +- .../storage/parquet/TestSchemaConverter.java | 10 +- .../org/apache/tajo/storage/s3/INode.java | 0 .../storage/s3/InMemoryFileSystemStore.java | 3 +- .../tajo/storage/s3/S3OutputStream.java | 0 .../storage/s3/SmallBlockS3FileSystem.java | 2 +- .../src/test/resources/storage-default.xml | 10 + .../src/test/resources/testVariousTypes.avsc | 0 tajo-yarn-pullserver/pom.xml | 7 +- 192 files changed, 1932 insertions(+), 1652 deletions(-) delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestHeapTuple.java delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestResizableSpec.java create mode 100644 tajo-storage/tajo-storage-common/pom.xml rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/Appender.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/DataLocation.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/DiskInfo.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/DiskMountInfo.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/DiskUtil.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/FrameTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/LazyTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/MemoryUtil.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/MergeScanner.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/NullScanner.java (59%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/NumericPathComparator.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/RowStoreUtil.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/Scanner.java (87%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/SeekableScanner.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/SerializerDeserializer.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/StorageManager.java (91%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/StorageProperty.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/StorageUtil.java (83%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/TableStatistics.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/TupleComparator.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/TupleRange.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/compress/CodecPool.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/exception/AlreadyExistsStorageException.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/exception/UnknownCodecException.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/exception/UnknownDataTypeException.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/exception/UnsupportedFileTypeException.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/fragment/Fragment.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/RowBlockReader.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/TupleBuilder.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/proto/IndexProtos.proto (100%) rename tajo-storage/{ => tajo-storage-common}/src/main/resources/storage-default.xml (94%) rename tajo-storage/{ => tajo-storage-common}/src/test/java/org/apache/tajo/storage/TestFrameTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/test/java/org/apache/tajo/storage/TestLazyTuple.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/test/java/org/apache/tajo/storage/TestTupleComparator.java (100%) rename tajo-storage/{ => tajo-storage-common}/src/test/java/org/apache/tajo/storage/TestVTuple.java (99%) create mode 100644 tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml create mode 100644 tajo-storage/tajo-storage-hbase/pom.xml rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java (94%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java (98%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java (98%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java (98%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java (100%) rename tajo-storage/{ => tajo-storage-hbase}/src/main/proto/StorageFragmentProtos.proto (95%) rename tajo-storage/{ => tajo-storage-hbase}/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java (92%) rename tajo-storage/{ => tajo-storage-hbase}/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java (100%) create mode 100644 tajo-storage/tajo-storage-hdfs/pom.xml rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/CSVFile.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/FileAppender.java (94%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/FileScanner.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/FileStorageManager.java (96%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java (97%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/LineReader.java (98%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/RawFile.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/RowFile.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/SplitLineReader.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/avro/package-info.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java (98%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/index/IndexMethod.java (88%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/index/IndexReader.java (97%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/index/IndexWriter.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/index/OrderIndexReader.java (95%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/TajoParquetReader.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/TajoParquetWriter.java (97%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java (93%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java (98%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/TajoRecordMaterializer.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java (89%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java (95%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/parquet/package-info.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/BytesRefArrayWritable.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/BytesRefWritable.java (97%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/LazyDecompressionCallback.java (94%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayInputStream.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayOutputStream.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataInputBuffer.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataOutputBuffer.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java (98%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionInputStream.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionOutputStream.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java (95%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java (98%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordReader.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java (95%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetReader.java (94%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java (90%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java (100%) rename tajo-storage/{src/main/java/org/apache/tajo/storage/Storage.java => tajo-storage-hdfs/src/main/proto/StorageFragmentProtos.proto} (54%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java (97%) rename tajo-storage/{src/test/java/org/apache/tajo/storage/TestStorageManager.java => tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java} (91%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/TestFileSystems.java (97%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/TestMergeScanner.java (98%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/TestStorages.java (93%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java (91%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java (97%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/parquet/TestReadWrite.java (95%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/parquet/TestSchemaConverter.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/s3/INode.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/s3/InMemoryFileSystemStore.java (99%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/s3/S3OutputStream.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/java/org/apache/tajo/storage/s3/SmallBlockS3FileSystem.java (100%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/resources/storage-default.xml (93%) rename tajo-storage/{ => tajo-storage-hdfs}/src/test/resources/testVariousTypes.avsc (100%) diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 60106065e4..4949355005 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -73,15 +73,6 @@ message FragmentProto { required bytes contents = 3; } -message FileFragmentProto { - required string id = 1; - required string path = 2; - required int64 startOffset = 3; - required int64 length = 4; - repeated string hosts = 7; - repeated int32 diskIds = 8; -} - message TableProto { required StoreType storeType = 1; optional KeyValueSetProto params = 2; diff --git a/tajo-client/pom.xml b/tajo-client/pom.xml index 7fc8c74b1e..c023db89d7 100644 --- a/tajo-client/pom.xml +++ b/tajo-client/pom.xml @@ -195,7 +195,11 @@ org.apache.tajo - tajo-storage + tajo-storage-common + + + org.apache.tajo + tajo-storage-hdfs org.apache.tajo diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java index b75e924829..2ad3268b70 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java @@ -30,10 +30,7 @@ import org.apache.tajo.client.QueryClient; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.FileScanner; -import org.apache.tajo.storage.MergeScanner; -import org.apache.tajo.storage.Scanner; -import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index b58ae89d29..2162dfd5ed 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -245,7 +245,16 @@ org.apache.tajo - tajo-storage + tajo-storage-common + + + org.apache.tajo + tajo-storage-hdfs + + + org.apache.tajo + tajo-storage-hbase + test org.apache.tajo @@ -483,7 +492,6 @@ org.apache.tajo tajo-hcatalog test - ${tajo.version} com.google.protobuf diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 66759e4867..091308e027 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -29,13 +29,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.SessionVars; -import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.physical.*; @@ -49,11 +47,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -921,7 +915,7 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St if (broadcastFlag) { PartitionedTableScanNode partitionedTableScanNode = (PartitionedTableScanNode) scanNode; List fileFragments = TUtil.newList(); - FileStorageManager fileStorageManager = StorageManager.getFileStorageManager(ctx.getConf()); + FileStorageManager fileStorageManager = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); for (Path path : partitionedTableScanNode.getInputPaths()) { fileFragments.addAll(TUtil.newList(fileStorageManager.split(scanNode.getCanonicalName(), path))); } @@ -1186,8 +1180,8 @@ public PhysicalExec createIndexScanExec(TaskAttemptContext ctx, FragmentConvertor.convert(ctx.getConf(), fragmentProtos); String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys()); - Path indexPath = new Path( - StorageManager.getFileStorageManager(ctx.getConf()).getTablePath(annotation.getTableName()), "index"); + FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); + Path indexPath = new Path(sm.getTablePath(annotation.getTableName()), "index"); TupleComparator comp = new BaseTupleComparator(annotation.getKeySchema(), annotation.getSortKeys()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index c5df5f916a..8ee4e2fadd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -35,6 +35,7 @@ import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.StoreTableNode; import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.unit.StorageUnit; @@ -161,7 +162,7 @@ public void openAppender(int suffixId) throws IOException { actualFilePath = new Path(lastFileName + "_" + suffixId); } - appender = StorageManager.getFileStorageManager(context.getConf()) + appender = ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 0538813843..51c4832bdc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -34,11 +34,7 @@ import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index 568c6ec23f..585d73aa22 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -80,8 +80,8 @@ public void init() throws IOException { context.getDataChannel().getStoreType() : CatalogProtos.StoreType.RAW); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); - this.appender = (FileAppender) StorageManager.getFileStorageManager(context.getConf()).getAppender(meta, - outSchema, new Path(storeTablePath, "output")); + this.appender = (FileAppender) ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) + .getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(); this.appender.init(); this.indexWriter = bst.getIndexWriter(new Path(storeTablePath, "index"), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index a5e0b5d370..3d3da5c7cf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -31,6 +31,7 @@ import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.unit.StorageUnit; @@ -92,7 +93,8 @@ public void openNewFile(int suffixId) throws IOException { lastFileName = new Path(lastFileName + "_" + suffixId); } - appender = StorageManager.getFileStorageManager(context.getConf()).getAppender(meta, appenderSchema, lastFileName); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) + .getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { LOG.info(prevFile + " exceeds " + SessionVars.MAX_OUTPUT_FILE_SIZE.keyname() + " (" + maxPerFileSize + " MB), " + diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java index aff4b7d7b1..25c0111932 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java @@ -28,7 +28,6 @@ import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryUnitRequest; @@ -40,8 +39,8 @@ import org.apache.tajo.master.querymaster.QueryUnit; import org.apache.tajo.master.querymaster.QueryUnitAttempt; import org.apache.tajo.master.querymaster.SubQuery; -import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; import org.apache.tajo.worker.FetchImpl; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index fe6291b4af..b75de04185 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -160,7 +160,7 @@ public void serviceInit(Configuration _conf) throws Exception { // check the system directory and create if they are not created. checkAndInitializeSystemDirectories(); - this.storeManager = StorageManager.getFileStorageManager(systemConf, null); + this.storeManager = (FileStorageManager)StorageManager.getFileStorageManager(systemConf, null); catalogServer = new CatalogServer(FunctionLoader.load()); addIfService(catalogServer); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 79800c2cdf..57426fea16 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -42,7 +42,6 @@ import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.LogicalRootNode; import org.apache.tajo.plan.rewrite.RewriteRule; -import org.apache.tajo.storage.hbase.AddSortForInsertRewriter; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.plan.logical.LogicalNode; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index b5119fdfb2..1788261373 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -97,7 +97,8 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC for (int i = 0; i < scans.length; i++) { TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName()); if (tableDesc == null) { // if it is a real table stored on storage - FileStorageManager storageManager = StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + FileStorageManager storageManager = + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); tablePath = storageManager.getTablePath(scans[i].getTableName()); if (execBlock.getUnionScanMap() != null && !execBlock.getUnionScanMap().isEmpty()) { @@ -410,7 +411,8 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster Path[] partitionScanPaths = null; TableDesc tableDesc = masterContext.getTableDescMap().get(eachScan.getCanonicalName()); if (eachScan.getType() == NodeType.PARTITIONS_SCAN) { - FileStorageManager storageManager = StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + FileStorageManager storageManager = + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)eachScan; partitionScanPaths = partitionScan.getInputPaths(); @@ -535,7 +537,7 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch partitionScanPaths = partitionScan.getInputPaths(); // set null to inputPaths in getFragmentsFromPartitionedTable() FileStorageManager storageManager = - StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); scanFragments = getFragmentsFromPartitionedTable(storageManager, scan, desc); } else { StorageManager storageManager = @@ -644,7 +646,8 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo ExecutionBlock execBlock = subQuery.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = StorageManager.getFileStorageManager(subQuery.getContext().getConf()).getTablePath(scan.getTableName()); + tablePath = ((FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf())) + .getTablePath(scan.getTableName()); ExecutionBlock sampleChildBlock = masterPlan.getChild(subQuery.getId(), 0); SortNode sortNode = PlannerUtil.findTopNode(sampleChildBlock.getPlan(), NodeType.SORT); @@ -809,7 +812,8 @@ public static void scheduleHashShuffledFetches(TaskSchedulerContext schedulerCon ExecutionBlock execBlock = subQuery.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = StorageManager.getFileStorageManager(subQuery.getContext().getConf()).getTablePath(scan.getTableName()); + tablePath = ((FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf())) + .getTablePath(scan.getTableName()); Fragment frag = new FileFragment(scan.getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); List fragments = new ArrayList(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 067d07bc52..8fc73c756f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -978,7 +978,7 @@ private static void scheduleFragmentsForLeafQuery(SubQuery subQuery) throws IOEx if (scan.getType() == NodeType.PARTITIONS_SCAN) { // After calling this method, partition paths are removed from the physical plan. FileStorageManager storageManager = - StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); fragments = Repartitioner.getFragmentsFromPartitionedTable(storageManager, scan, table); } else { StorageManager storageManager = diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 4bcae5efaf..390740ca30 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -51,11 +51,7 @@ import org.apache.tajo.pullserver.TajoPullServerService; import org.apache.tajo.pullserver.retriever.FileChunk; import org.apache.tajo.rpc.NullCallback; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.HashShuffleAppenderManager; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.StorageUtil; -import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.NetUtils; import org.jboss.netty.channel.socket.ClientSocketChannelFactory; @@ -156,8 +152,8 @@ public Task(String taskRunnerId, this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys()); } } else { - Path outFilePath = StorageManager.getFileStorageManager(systemConf).getAppenderFilePath( - taskId, queryContext.getStagingDir()); + Path outFilePath = ((FileStorageManager)StorageManager.getFileStorageManager(systemConf)) + .getAppenderFilePath(taskId, queryContext.getStagingDir()); LOG.info("Output File Path: " + outFilePath); context.setOutputPath(outFilePath); } diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index 0959f2ead1..fb98be2756 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -47,7 +47,7 @@ public class BackendTestingUtil { public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { - FileStorageManager sm = StorageManager.getFileStorageManager(conf, tablePath); + FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, tablePath); FileSystem fs = sm.getFileSystem(); Appender appender; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index fee6f4c48a..f1501c6ed4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -40,10 +40,10 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 64d7cee1d5..3803c7a3c6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -42,10 +42,7 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.*; import org.apache.tajo.util.CommonTestingUtil; import org.junit.After; import org.junit.Before; @@ -140,8 +137,8 @@ private TableDesc makeTestData(String tableName, Schema schema, int dataSize) th contentsData += j; } } - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(tableMeta, schema, - dataPath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(tableMeta, schema, dataPath); appender.init(); Tuple tuple = new VTuple(schema.size()); int writtenSize = 0; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index a1d657f755..e1f7333aa6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -89,7 +89,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < OUTER_TUPLE_NUM; i++) { @@ -110,7 +111,8 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < INNER_TUPLE_NUM; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 0a58638d85..634f6a5f78 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -91,7 +91,7 @@ public void setup() throws Exception { Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = StorageManager.getFileStorageManager(conf, workDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir); idxPath = new Path(workDir, "test.idx"); @@ -117,8 +117,7 @@ public void setup() throws Exception { fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); - FileAppender appender = (FileAppender)StorageManager.getFileStorageManager(conf).getAppender(meta, schema, - tablePath); + FileAppender appender = (FileAppender)sm.getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 10000; i++) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 89b623ba34..9f7038c182 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -76,7 +76,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = StorageManager.getFileStorageManager(conf, testDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -85,7 +85,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index dfabe18788..3ca9ba0a81 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -84,7 +84,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -107,7 +107,8 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -136,7 +137,8 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -175,7 +177,8 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -227,8 +230,8 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index f3c0294675..7439a72043 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -111,7 +111,8 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -149,7 +150,8 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManager.getFileStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -180,7 +182,8 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -219,7 +222,8 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -271,8 +275,8 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); appender5.close(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 760a47041e..a9922b5525 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -86,8 +86,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -112,7 +112,8 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index c3604a5bee..89740d49e3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -88,8 +88,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); for (int i = 0; i < 10; i++) { @@ -111,7 +111,8 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 26507de4e7..f979596971 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -86,8 +86,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -112,7 +112,8 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); // make 27 tuples diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 9e3226bf9e..24c17bcb81 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -108,7 +108,8 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -137,7 +138,8 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -176,7 +178,8 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -228,8 +231,8 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index bc13e06bcd..fc0bb0e3e9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -104,7 +104,8 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -133,7 +134,8 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -172,7 +174,8 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -224,8 +227,8 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index ff8e1d9699..7655164c36 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -89,8 +89,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); for (int i = 0; i < 10; i++) { @@ -118,7 +118,8 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 4cead8293f..48ec855145 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -86,7 +86,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 50; i++) { @@ -109,7 +110,8 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 50; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 06f4001bfa..7ec7a24c24 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -104,7 +104,7 @@ public static void setUp() throws Exception { util.startCatalogCluster(); conf = util.getConfiguration(); testDir = CommonTestingUtil.getTestDir("target/test-data/TestPhysicalPlanner"); - sm = StorageManager.getFileStorageManager(conf, testDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); @@ -127,8 +127,7 @@ public static void setUp() throws Exception { Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = sm.getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); for (int i = 0; i < 100; i++) { @@ -146,7 +145,7 @@ public static void setUp() throws Exception { Path scorePath = new Path(testDir, "score"); TableMeta scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV, new KeyValueSet()); - appender = StorageManager.getFileStorageManager(conf).getAppender(scoreMeta, scoreSchema, scorePath); + appender = sm.getAppender(scoreMeta, scoreSchema, scorePath); appender.init(); score = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta, @@ -187,8 +186,8 @@ public static void createLargeScoreTable() throws IOException { Schema scoreSchmea = score.getSchema(); TableMeta scoreLargeMeta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(scoreLargeMeta, scoreSchmea, - scoreLargePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init(); largeScore = new TableDesc( @@ -449,8 +448,8 @@ public final void testStorePlan() throws IOException, PlanningException { exec.next(); exec.close(); - Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), - ctx.getOutputPath()); + Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; int i = 0; @@ -509,7 +508,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann // checking the file contents long totalNum = 0; for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { - Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner( + Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)).getFileScanner( CatalogUtil.newTableMeta(StoreType.CSV), rootNode.getOutSchema(), status.getPath()); @@ -546,7 +545,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio exec.next(); exec.close(); - Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), + Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)).getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -1081,7 +1080,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { Path outputPath = StorageUtil.concatPath(workDir, "output", "output"); TableMeta meta = CatalogUtil.newTableMeta(channel.getStoreType(), new KeyValueSet()); SeekableScanner scanner = - StorageManager.getSeekableScanner(conf, meta, exec.getSchema(), outputPath); + FileStorageManager.getSeekableScanner(conf, meta, exec.getSchema(), outputPath); scanner.init(); int cnt = 0; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index 312da9e714..110ea9cb0c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -89,7 +89,8 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.RAW); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 6f88b3022f..6d2e7a9a01 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -103,7 +103,8 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -132,7 +133,8 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -171,7 +173,8 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 35238f48b0..c170cc69c1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -110,7 +110,8 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -148,7 +149,8 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManager.getFileStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -179,7 +181,8 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -218,7 +221,8 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -270,8 +274,8 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 42c0c15f8a..1d28ed2858 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -70,7 +70,7 @@ public static void setUp() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); - sm = StorageManager.getFileStorageManager(conf, workDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -82,7 +82,8 @@ public static void setUp() throws Exception { tablePath = StorageUtil.concatPath(workDir, "employee", "table1"); sm.getFileSystem().mkdirs(tablePath.getParent()); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, tablePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 100; i++) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index 343943fa9b..b743da882b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -565,8 +565,8 @@ private void createMultiFile(String tableName, int numRowsEachFile, TupleCreator } Path dataPath = new Path(table.getPath(), fileIndex + ".csv"); fileIndex++; - appender = StorageManager.getFileStorageManager(conf).getAppender(tableMeta, schema, - dataPath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(tableMeta, schema, dataPath); appender.init(); } String[] columnDatas = rows[i].split("\\|"); diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index ab0979b408..f2ec804d09 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -63,7 +63,7 @@ public class TestResultSet { public static void setup() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); scoreSchema = new Schema(); scoreSchema.addColumn("deptname", Type.TEXT); @@ -73,8 +73,7 @@ public static void setup() throws Exception { Path p = sm.getTablePath("score"); sm.getFileSystem().mkdirs(p); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(scoreMeta, scoreSchema, - new Path(p, "score")); + Appender appender = sm.getAppender(scoreMeta, scoreSchema, new Path(p, "score")); appender.init(); int deptSize = 100; int tupleNum = 10000; diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java index 2aa56db4b4..f36ff24531 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -36,7 +36,6 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.FileUtil; import org.junit.After; import org.junit.Before; @@ -70,7 +69,8 @@ public void test() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(StoreType.ROWFILE); - FileStorageManager sm = StorageManager.getFileStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR))); + FileStorageManager sm = + (FileStorageManager)StorageManager.getFileStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR))); Path tablePath = new Path("/test"); Path metaPath = new Path(tablePath, ".meta"); @@ -80,7 +80,7 @@ public void test() throws IOException { FileUtil.writeProto(fs, metaPath, meta.getProto()); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, dataPath); + Appender appender = sm.getAppender(meta, schema, dataPath); appender.enableStats(); appender.init(); diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index a16d9f0d6a..16d0a51dff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -185,7 +185,7 @@ public void testGet() throws Exception { reader.open(); TableMeta meta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - SeekableScanner scanner = StorageManager.getSeekableScanner(conf, meta, schema, + SeekableScanner scanner = FileStorageManager.getSeekableScanner(conf, meta, schema, StorageUtil.concatPath(testDir, "output", "output")); scanner.init(); @@ -308,7 +308,7 @@ public void testGetFromDescendingOrder() throws Exception { new Path(testDir, "output/index"), keySchema, comp); reader.open(); TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - SeekableScanner scanner = StorageManager.getSeekableScanner(conf, outputMeta, schema, + SeekableScanner scanner = FileStorageManager.getSeekableScanner(conf, outputMeta, schema, StorageUtil.concatPath(testDir, "output", "output")); scanner.init(); int cnt = 0; diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml index d350889bf0..eb8ada9fa6 100644 --- a/tajo-dist/pom.xml +++ b/tajo-dist/pom.xml @@ -75,7 +75,12 @@ org.apache.tajo - tajo-storage + tajo-storage-common + provided + + + org.apache.tajo + tajo-storage-hdfs provided diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index 20cdf16e94..1c3c4102b0 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -102,7 +102,11 @@ org.apache.tajo - tajo-storage + tajo-storage-common + + + org.apache.tajo + tajo-storage-hdfs org.apache.tajo diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index df8e814bf5..6881d40087 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -744,6 +744,22 @@ org.apache.tajo tajo-storage ${tajo.version} + pom + + + org.apache.tajo + tajo-storage-common + ${tajo.version} + + + org.apache.tajo + tajo-storage-hdfs + ${tajo.version} + + + org.apache.tajo + tajo-storage-hbase + ${tajo.version} org.apache.tajo diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 5a265e8ce9..8acb1a9b92 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -16,329 +16,47 @@ --> - 4.0.0 tajo-project org.apache.tajo 0.9.1-SNAPSHOT ../tajo-project - + 4.0.0 tajo-storage - jar + pom Tajo Storage - Tajo Storage Package - UTF-8 UTF-8 - 1.5.0 - 2.1.0 - - - repository.jboss.org - https://repository.jboss.org/nexus/content/repositories/releases/ - - - false - - - + + tajo-storage-common + tajo-storage-hdfs + tajo-storage-hbase + - - org.apache.maven.plugins - maven-compiler-plugin - - 1.6 - 1.6 - ${project.build.sourceEncoding} - - org.apache.rat apache-rat-plugin - - - verify - - check - - - - - - src/test/resources/testVariousTypes.avsc - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - TRUE - - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - + maven-surefire-report-plugin - org.apache.maven.plugins - maven-antrun-plugin - - - create-protobuf-generated-sources-directory - initialize - - - - - - - run - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.2 - - - generate-sources - generate-sources - - protoc - - -Isrc/main/proto/ - --proto_path=../tajo-common/src/main/proto - --proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto - --java_out=target/generated-sources/proto - src/main/proto/IndexProtos.proto - src/main/proto/StorageFragmentProtos.proto - - - - exec - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 1.5 - - - add-source - generate-sources - - add-source - - - - target/generated-sources/proto - - - - - - - org.apache.maven.plugins - maven-pmd-plugin - 2.7.1 + maven-deploy-plugin + + true + - - - org.apache.tajo - tajo-common - - - org.apache.tajo - tajo-catalog-common - - - org.apache.tajo - tajo-plan - - - - org.apache.avro - trevni-core - 1.7.3 - - - org.apache.avro - trevni-avro - 1.7.3 - - - org.apache.hadoop - hadoop-core - - - - - org.apache.hadoop - hadoop-common - provided - - - zookeeper - org.apache.zookeeper - - - slf4j-api - org.slf4j - - - jersey-json - com.sun.jersey - - - - - org.apache.hadoop - hadoop-hdfs - provided - - - commons-el - commons-el - - - tomcat - jasper-runtime - - - tomcat - jasper-compiler - - - org.mortbay.jetty - jsp-2.1-jetty - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - - - - - org.apache.hadoop - hadoop-minicluster - test - - - commons-el - commons-el - - - tomcat - jasper-runtime - - - tomcat - jasper-compiler - - - org.mortbay.jetty - jsp-2.1-jetty - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - - - hadoop-yarn-server-tests - org.apache.hadoop - - - hadoop-mapreduce-client-jobclient - org.apache.hadoop - - - hadoop-mapreduce-client-app - org.apache.hadoop - - - hadoop-yarn-api - org.apache.hadoop - - - hadoop-mapreduce-client-hs - org.apache.hadoop - - - hadoop-mapreduce-client-core - org.apache.hadoop - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - provided - - - com.google.protobuf - protobuf-java - - - junit - junit - test - - - com.twitter - parquet-column - ${parquet.version} - - - com.twitter - parquet-hadoop - ${parquet.version} - - - com.twitter - parquet-format - ${parquet.format.version} - - - org.apache.hbase - hbase-server - ${hbase.version} - provided - - - org.apache.hbase - hbase-client - ${hbase.version} - provided - - - docs @@ -383,7 +101,7 @@ dist - package + prepare-package run @@ -406,12 +124,15 @@ echo echo "Current directory `pwd`" echo - run rm -rf ${project.artifactId}-${project.version} - run mkdir ${project.artifactId}-${project.version} - run cd ${project.artifactId}-${project.version} - run cp -r ${basedir}/target/${project.artifactId}-${project.version}*.jar . + run rm -rf tajo-storage-${project.version} + run mkdir tajo-storage-${project.version} + run cd tajo-storage-${project.version} + run cp -r ${basedir}/tajo-storage-common/target/tajo-storage-common-${project.version}*.jar . + run cp -r ${basedir}/tajo-storage-hdfs/target/tajo-storage-hdfs-${project.version}*.jar . + run cp -r ${basedir}/tajo-storage-hbase/target/tajo-storage-hbase-${project.version}*.jar . + echo - echo "Tajo Storage dist layout available at: ${project.build.directory}/${project.artifactId}-${project.version}" + echo "Tajo Storage dist layout available at: ${project.build.directory}/tajo-storage-${project.version}" echo @@ -431,11 +152,7 @@ org.apache.maven.plugins - maven-project-info-reports-plugin - 2.4 - - false - + maven-surefire-report-plugin diff --git a/tajo-storage/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java b/tajo-storage/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java deleted file mode 100644 index b332364423..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/tuple/TestBaseTupleBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple; - -import org.apache.tajo.storage.RowStoreUtil; -import org.apache.tajo.tuple.offheap.*; -import org.junit.Test; - -public class TestBaseTupleBuilder { - - @Test - public void testBuild() { - BaseTupleBuilder builder = new BaseTupleBuilder(TestOffHeapRowBlock.schema); - - OffHeapRowBlock rowBlock = TestOffHeapRowBlock.createRowBlock(10248); - OffHeapRowBlockReader reader = rowBlock.getReader(); - - ZeroCopyTuple inputTuple = new ZeroCopyTuple(); - - HeapTuple heapTuple = null; - ZeroCopyTuple zcTuple = null; - int i = 0; - while(reader.next(inputTuple)) { - RowStoreUtil.convert(inputTuple, builder); - - heapTuple = builder.buildToHeapTuple(); - TestOffHeapRowBlock.validateTupleResult(i, heapTuple); - - zcTuple = builder.buildToZeroCopyTuple(); - TestOffHeapRowBlock.validateTupleResult(i, zcTuple); - - i++; - } - } - - @Test - public void testBuildWithNull() { - BaseTupleBuilder builder = new BaseTupleBuilder(TestOffHeapRowBlock.schema); - - OffHeapRowBlock rowBlock = TestOffHeapRowBlock.createRowBlockWithNull(10248); - OffHeapRowBlockReader reader = rowBlock.getReader(); - - ZeroCopyTuple inputTuple = new ZeroCopyTuple(); - - HeapTuple heapTuple = null; - ZeroCopyTuple zcTuple = null; - int i = 0; - while(reader.next(inputTuple)) { - RowStoreUtil.convert(inputTuple, builder); - - heapTuple = builder.buildToHeapTuple(); - TestOffHeapRowBlock.validateNullity(i, heapTuple); - - zcTuple = builder.buildToZeroCopyTuple(); - TestOffHeapRowBlock.validateNullity(i, zcTuple); - - i++; - } - } -} \ No newline at end of file diff --git a/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestHeapTuple.java b/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestHeapTuple.java deleted file mode 100644 index 96f465a59e..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestHeapTuple.java +++ /dev/null @@ -1,45 +0,0 @@ -/*** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple.offheap; - -import org.apache.tajo.catalog.SchemaUtil; -import org.junit.Test; - -public class TestHeapTuple { - - @Test - public void testHeapTuple() { - OffHeapRowBlock rowBlock = TestOffHeapRowBlock.createRowBlock(1024); - - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - - ZeroCopyTuple zcTuple = new ZeroCopyTuple(); - int i = 0; - while (reader.next(zcTuple)) { - byte [] bytes = new byte[zcTuple.nioBuffer().limit()]; - zcTuple.nioBuffer().get(bytes); - - HeapTuple heapTuple = new HeapTuple(bytes, SchemaUtil.toDataTypes(TestOffHeapRowBlock.schema)); - TestOffHeapRowBlock.validateTupleResult(i, heapTuple); - i++; - } - - rowBlock.release(); - } -} \ No newline at end of file diff --git a/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java b/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java deleted file mode 100644 index c43ba38930..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestOffHeapRowBlock.java +++ /dev/null @@ -1,577 +0,0 @@ -/*** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple.offheap; - -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.*; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.ProtobufDatum; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.RowStoreUtil; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; -import org.apache.tajo.unit.StorageUnit; -import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.ProtoUtil; -import org.junit.Test; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; - -import static org.apache.tajo.common.TajoDataTypes.Type; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TestOffHeapRowBlock { - private static final Log LOG = LogFactory.getLog(TestOffHeapRowBlock.class); - public static String UNICODE_FIELD_PREFIX = "abc_가나다_"; - public static Schema schema; - - static { - schema = new Schema(); - schema.addColumn("col0", Type.BOOLEAN); - schema.addColumn("col1", Type.INT2); - schema.addColumn("col2", Type.INT4); - schema.addColumn("col3", Type.INT8); - schema.addColumn("col4", Type.FLOAT4); - schema.addColumn("col5", Type.FLOAT8); - schema.addColumn("col6", Type.TEXT); - schema.addColumn("col7", Type.TIMESTAMP); - schema.addColumn("col8", Type.DATE); - schema.addColumn("col9", Type.TIME); - schema.addColumn("col10", Type.INTERVAL); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", - CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, PrimitiveProtos.StringProto.class.getName())); - } - - private void explainRowBlockAllocation(OffHeapRowBlock rowBlock, long startTime, long endTime) { - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " - + (endTime - startTime) + " msec"); - } - - @Test - public void testPutAndReadValidation() { - int rowNum = 1000; - - long allocStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, 1024); - long allocEnd = System.currentTimeMillis(); - explainRowBlockAllocation(rowBlock, allocStart, allocEnd); - - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - - ZeroCopyTuple tuple = new ZeroCopyTuple(); - long writeStart = System.currentTimeMillis(); - for (int i = 0; i < rowNum; i++) { - fillRow(i, rowBlock.getWriter()); - - reader.reset(); - int j = 0; - while(reader.next(tuple)) { - validateTupleResult(j, tuple); - - j++; - } - } - long writeEnd = System.currentTimeMillis(); - LOG.info("writing and validating take " + (writeEnd - writeStart) + " msec"); - - long readStart = System.currentTimeMillis(); - tuple = new ZeroCopyTuple(); - int j = 0; - reader.reset(); - while(reader.next(tuple)) { - validateTupleResult(j, tuple); - j++; - } - long readEnd = System.currentTimeMillis(); - LOG.info("reading takes " + (readEnd - readStart) + " msec"); - - rowBlock.release(); - } - - @Test - public void testNullityValidation() { - int rowNum = 1000; - - long allocStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, 1024); - long allocEnd = System.currentTimeMillis(); - explainRowBlockAllocation(rowBlock, allocStart, allocEnd); - - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - ZeroCopyTuple tuple = new ZeroCopyTuple(); - long writeStart = System.currentTimeMillis(); - for (int i = 0; i < rowNum; i++) { - fillRowBlockWithNull(i, rowBlock.getWriter()); - - reader.reset(); - int j = 0; - while(reader.next(tuple)) { - validateNullity(j, tuple); - - j++; - } - } - long writeEnd = System.currentTimeMillis(); - LOG.info("writing and nullity validating take " + (writeEnd - writeStart) +" msec"); - - long readStart = System.currentTimeMillis(); - tuple = new ZeroCopyTuple(); - int j = 0; - reader.reset(); - while(reader.next(tuple)) { - validateNullity(j, tuple); - - j++; - } - long readEnd = System.currentTimeMillis(); - LOG.info("reading takes " + (readEnd - readStart) + " msec"); - - rowBlock.release(); - } - - @Test - public void testEmptyRow() { - int rowNum = 1000; - - long allocStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 10); - long allocEnd = System.currentTimeMillis(); - explainRowBlockAllocation(rowBlock, allocStart, allocEnd); - - long writeStart = System.currentTimeMillis(); - for (int i = 0; i < rowNum; i++) { - rowBlock.getWriter().startRow(); - // empty columns - rowBlock.getWriter().endRow(); - } - long writeEnd = System.currentTimeMillis(); - LOG.info("writing tooks " + (writeEnd - writeStart) + " msec"); - - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - - long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); - int j = 0; - reader.reset(); - while(reader.next(tuple)) { - j++; - } - long readEnd = System.currentTimeMillis(); - LOG.info("reading takes " + (readEnd - readStart) + " msec"); - rowBlock.release(); - - assertEquals(rowNum, j); - assertEquals(rowNum, rowBlock.rows()); - } - - @Test - public void testSortBenchmark() { - int rowNum = 1000; - - OffHeapRowBlock rowBlock = createRowBlock(rowNum); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - - List unSafeTuples = Lists.newArrayList(); - - long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); - reader.reset(); - while(reader.next(tuple)) { - unSafeTuples.add(tuple); - tuple = new ZeroCopyTuple(); - } - long readEnd = System.currentTimeMillis(); - LOG.info("reading takes " + (readEnd - readStart) + " msec"); - - SortSpec sortSpec = new SortSpec(new Column("col2", Type.INT4)); - BaseTupleComparator comparator = new BaseTupleComparator(schema, new SortSpec[] {sortSpec}); - - long sortStart = System.currentTimeMillis(); - Collections.sort(unSafeTuples, comparator); - long sortEnd = System.currentTimeMillis(); - LOG.info("sorting took " + (sortEnd - sortStart) + " msec"); - rowBlock.release(); - } - - @Test - public void testVTuplePutAndGetBenchmark() { - int rowNum = 1000; - - List rowBlock = Lists.newArrayList(); - long writeStart = System.currentTimeMillis(); - VTuple tuple; - for (int i = 0; i < rowNum; i++) { - tuple = new VTuple(schema.size()); - fillVTuple(i, tuple); - rowBlock.add(tuple); - } - long writeEnd = System.currentTimeMillis(); - LOG.info("Writing takes " + (writeEnd - writeStart) + " msec"); - - long readStart = System.currentTimeMillis(); - int j = 0; - for (VTuple t : rowBlock) { - validateTupleResult(j, t); - j++; - } - long readEnd = System.currentTimeMillis(); - LOG.info("reading takes " + (readEnd - readStart) + " msec"); - - int count = 0; - for (int l = 0; l < rowBlock.size(); l++) { - for(int m = 0; m < schema.size(); m++ ) { - if (rowBlock.get(l).contains(m) && rowBlock.get(l).get(m).type() == Type.INT4) { - count ++; - } - } - } - // For preventing unnecessary code elimination optimization. - LOG.info("The number of INT4 values is " + count + "."); - } - - @Test - public void testVTuplePutAndGetBenchmarkViaDirectRowEncoder() { - int rowNum = 1000; - - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 100); - - long writeStart = System.currentTimeMillis(); - VTuple tuple = new VTuple(schema.size()); - for (int i = 0; i < rowNum; i++) { - fillVTuple(i, tuple); - - RowStoreUtil.convert(tuple, rowBlock.getWriter()); - } - long writeEnd = System.currentTimeMillis(); - LOG.info("Writing takes " + (writeEnd - writeStart) + " msec"); - - validateResults(rowBlock); - rowBlock.release(); - } - - @Test - public void testSerDerOfRowBlock() { - int rowNum = 1000; - - OffHeapRowBlock rowBlock = createRowBlock(rowNum); - - ByteBuffer bb = rowBlock.nioBuffer(); - OffHeapRowBlock restoredRowBlock = new OffHeapRowBlock(schema, bb); - validateResults(restoredRowBlock); - rowBlock.release(); - } - - @Test - public void testSerDerOfZeroCopyTuple() { - int rowNum = 1000; - - OffHeapRowBlock rowBlock = createRowBlock(rowNum); - - ByteBuffer bb = rowBlock.nioBuffer(); - OffHeapRowBlock restoredRowBlock = new OffHeapRowBlock(schema, bb); - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(restoredRowBlock); - - long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); - ZeroCopyTuple copyTuple = new ZeroCopyTuple(); - int j = 0; - reader.reset(); - while(reader.next(tuple)) { - ByteBuffer copy = tuple.nioBuffer(); - copyTuple.set(copy, SchemaUtil.toDataTypes(schema)); - - validateTupleResult(j, copyTuple); - - j++; - } - long readEnd = System.currentTimeMillis(); - LOG.info("reading takes " + (readEnd - readStart) + " msec"); - - rowBlock.release(); - } - - public static OffHeapRowBlock createRowBlock(int rowNum) { - long allocateStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 8); - long allocatedEnd = System.currentTimeMillis(); - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " - + (allocatedEnd - allocateStart) + " msec"); - - long writeStart = System.currentTimeMillis(); - for (int i = 0; i < rowNum; i++) { - fillRow(i, rowBlock.getWriter()); - } - long writeEnd = System.currentTimeMillis(); - LOG.info("writing takes " + (writeEnd - writeStart) + " msec"); - - return rowBlock; - } - - public static OffHeapRowBlock createRowBlockWithNull(int rowNum) { - long allocateStart = System.currentTimeMillis(); - OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 8); - long allocatedEnd = System.currentTimeMillis(); - LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated " - + (allocatedEnd - allocateStart) + " msec"); - - long writeStart = System.currentTimeMillis(); - for (int i = 0; i < rowNum; i++) { - fillRowBlockWithNull(i, rowBlock.getWriter()); - } - long writeEnd = System.currentTimeMillis(); - LOG.info("writing and validating take " + (writeEnd - writeStart) + " msec"); - - return rowBlock; - } - - public static void fillRow(int i, RowWriter builder) { - builder.startRow(); - builder.putBool(i % 1 == 0 ? true : false); // 0 - builder.putInt2((short) 1); // 1 - builder.putInt4(i); // 2 - builder.putInt8(i); // 3 - builder.putFloat4(i); // 4 - builder.putFloat8(i); // 5 - builder.putText((UNICODE_FIELD_PREFIX + i).getBytes()); // 6 - builder.putTimestamp(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + i); // 7 - builder.putDate(DatumFactory.createDate("2014-04-16").asInt4() + i); // 8 - builder.putTime(DatumFactory.createTime("08:48:00").asInt8() + i); // 9 - builder.putInterval(DatumFactory.createInterval((i + 1) + " hours")); // 10 - builder.putInet4(DatumFactory.createInet4("192.168.0.1").asInt4() + i); // 11 - builder.putProtoDatum(new ProtobufDatum(ProtoUtil.convertString(i + ""))); // 12 - builder.endRow(); - } - - public static void fillRowBlockWithNull(int i, RowWriter writer) { - writer.startRow(); - - if (i == 0) { - writer.skipField(); - } else { - writer.putBool(i % 1 == 0 ? true : false); // 0 - } - if (i % 1 == 0) { - writer.skipField(); - } else { - writer.putInt2((short) 1); // 1 - } - - if (i % 2 == 0) { - writer.skipField(); - } else { - writer.putInt4(i); // 2 - } - - if (i % 3 == 0) { - writer.skipField(); - } else { - writer.putInt8(i); // 3 - } - - if (i % 4 == 0) { - writer.skipField(); - } else { - writer.putFloat4(i); // 4 - } - - if (i % 5 == 0) { - writer.skipField(); - } else { - writer.putFloat8(i); // 5 - } - - if (i % 6 == 0) { - writer.skipField(); - } else { - writer.putText((UNICODE_FIELD_PREFIX + i).getBytes()); // 6 - } - - if (i % 7 == 0) { - writer.skipField(); - } else { - writer.putTimestamp(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + i); // 7 - } - - if (i % 8 == 0) { - writer.skipField(); - } else { - writer.putDate(DatumFactory.createDate("2014-04-16").asInt4() + i); // 8 - } - - if (i % 9 == 0) { - writer.skipField(); - } else { - writer.putTime(DatumFactory.createTime("08:48:00").asInt8() + i); // 9 - } - - if (i % 10 == 0) { - writer.skipField(); - } else { - writer.putInterval(DatumFactory.createInterval((i + 1) + " hours")); // 10 - } - - if (i % 11 == 0) { - writer.skipField(); - } else { - writer.putInet4(DatumFactory.createInet4("192.168.0.1").asInt4() + i); // 11 - } - - if (i % 12 == 0) { - writer.skipField(); - } else { - writer.putProtoDatum(new ProtobufDatum(ProtoUtil.convertString(i + ""))); // 12 - } - - writer.endRow(); - } - - public static void fillVTuple(int i, VTuple tuple) { - tuple.put(0, DatumFactory.createBool(i % 1 == 0)); - tuple.put(1, DatumFactory.createInt2((short) 1)); - tuple.put(2, DatumFactory.createInt4(i)); - tuple.put(3, DatumFactory.createInt8(i)); - tuple.put(4, DatumFactory.createFloat4(i)); - tuple.put(5, DatumFactory.createFloat8(i)); - tuple.put(6, DatumFactory.createText((UNICODE_FIELD_PREFIX + i).getBytes())); - tuple.put(7, DatumFactory.createTimestamp(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + i)); // 7 - tuple.put(8, DatumFactory.createDate(DatumFactory.createDate("2014-04-16").asInt4() + i)); // 8 - tuple.put(9, DatumFactory.createTime(DatumFactory.createTime("08:48:00").asInt8() + i)); // 9 - tuple.put(10, DatumFactory.createInterval((i + 1) + " hours")); // 10 - tuple.put(11, DatumFactory.createInet4(DatumFactory.createInet4("192.168.0.1").asInt4() + i)); // 11 - tuple.put(12, new ProtobufDatum(ProtoUtil.convertString(i + ""))); // 12; - } - - public static void validateResults(OffHeapRowBlock rowBlock) { - long readStart = System.currentTimeMillis(); - ZeroCopyTuple tuple = new ZeroCopyTuple(); - int j = 0; - OffHeapRowBlockReader reader = new OffHeapRowBlockReader(rowBlock); - reader.reset(); - while(reader.next(tuple)) { - validateTupleResult(j, tuple); - j++; - } - long readEnd = System.currentTimeMillis(); - LOG.info("Reading takes " + (readEnd - readStart) + " msec"); - } - - public static void validateTupleResult(int j, Tuple t) { - assertTrue((j % 1 == 0) == t.getBool(0)); - assertTrue(1 == t.getInt2(1)); - assertEquals(j, t.getInt4(2)); - assertEquals(j, t.getInt8(3)); - assertTrue(j == t.getFloat4(4)); - assertTrue(j == t.getFloat8(5)); - assertEquals(new String(UNICODE_FIELD_PREFIX + j), t.getText(6)); - assertEquals(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + (long) j, t.getInt8(7)); - assertEquals(DatumFactory.createDate("2014-04-16").asInt4() + j, t.getInt4(8)); - assertEquals(DatumFactory.createTime("08:48:00").asInt8() + j, t.getInt8(9)); - assertEquals(DatumFactory.createInterval((j + 1) + " hours"), t.getInterval(10)); - assertEquals(DatumFactory.createInet4("192.168.0.1").asInt4() + j, t.getInt4(11)); - assertEquals(new ProtobufDatum(ProtoUtil.convertString(j + "")), t.getProtobufDatum(12)); - } - - public static void validateNullity(int j, Tuple tuple) { - if (j == 0) { - tuple.isNull(0); - } else { - assertTrue((j % 1 == 0) == tuple.getBool(0)); - } - - if (j % 1 == 0) { - tuple.isNull(1); - } else { - assertTrue(1 == tuple.getInt2(1)); - } - - if (j % 2 == 0) { - tuple.isNull(2); - } else { - assertEquals(j, tuple.getInt4(2)); - } - - if (j % 3 == 0) { - tuple.isNull(3); - } else { - assertEquals(j, tuple.getInt8(3)); - } - - if (j % 4 == 0) { - tuple.isNull(4); - } else { - assertTrue(j == tuple.getFloat4(4)); - } - - if (j % 5 == 0) { - tuple.isNull(5); - } else { - assertTrue(j == tuple.getFloat8(5)); - } - - if (j % 6 == 0) { - tuple.isNull(6); - } else { - assertEquals(new String(UNICODE_FIELD_PREFIX + j), tuple.getText(6)); - } - - if (j % 7 == 0) { - tuple.isNull(7); - } else { - assertEquals(DatumFactory.createTimestamp("2014-04-16 08:48:00").asInt8() + (long) j, tuple.getInt8(7)); - } - - if (j % 8 == 0) { - tuple.isNull(8); - } else { - assertEquals(DatumFactory.createDate("2014-04-16").asInt4() + j, tuple.getInt4(8)); - } - - if (j % 9 == 0) { - tuple.isNull(9); - } else { - assertEquals(DatumFactory.createTime("08:48:00").asInt8() + j, tuple.getInt8(9)); - } - - if (j % 10 == 0) { - tuple.isNull(10); - } else { - assertEquals(DatumFactory.createInterval((j + 1) + " hours"), tuple.getInterval(10)); - } - - if (j % 11 == 0) { - tuple.isNull(11); - } else { - assertEquals(DatumFactory.createInet4("192.168.0.1").asInt4() + j, tuple.getInt4(11)); - } - - if (j % 12 == 0) { - tuple.isNull(12); - } else { - assertEquals(new ProtobufDatum(ProtoUtil.convertString(j + "")), tuple.getProtobufDatum(12)); - } - } -} \ No newline at end of file diff --git a/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestResizableSpec.java b/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestResizableSpec.java deleted file mode 100644 index 1eb9c17d07..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/tuple/offheap/TestResizableSpec.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.tuple.offheap; - -import org.apache.tajo.unit.StorageUnit; -import org.junit.Test; - -import static org.junit.Assert.*; - -public class TestResizableSpec { - - @Test - public void testResizableLimit() { - ResizableLimitSpec limit = new ResizableLimitSpec(10 * StorageUnit.MB, 1000 * StorageUnit.MB, 0.1f, 1.0f); - - long expectedMaxSize = (long) (1000 * StorageUnit.MB + (1000 * StorageUnit.MB * 0.1f)); - - assertTrue(limit.limit() == 1000 * StorageUnit.MB + (1000 * StorageUnit.MB * 0.1f)); - - assertEquals(20971520, limit.increasedSize(10 * StorageUnit.MB)); - - assertEquals(expectedMaxSize, limit.increasedSize(1600 * StorageUnit.MB)); - - assertEquals(0.98f, limit.remainRatio(980 * StorageUnit.MB), 0.1); - - assertFalse(limit.canIncrease(limit.limit())); - } - - @Test - public void testFixedLimit() { - FixedSizeLimitSpec limit = new FixedSizeLimitSpec(100 * StorageUnit.MB, 0.0f); - - assertEquals(limit.limit(), 100 * StorageUnit.MB); - - assertEquals(100 * StorageUnit.MB, limit.increasedSize(1000)); - - assertEquals(100 * StorageUnit.MB, limit.increasedSize(1600 * StorageUnit.MB)); - - assertTrue(0.98f == limit.remainRatio(98 * StorageUnit.MB)); - - assertFalse(limit.canIncrease(limit.limit())); - } -} \ No newline at end of file diff --git a/tajo-storage/tajo-storage-common/pom.xml b/tajo-storage/tajo-storage-common/pom.xml new file mode 100644 index 0000000000..73dd6e2859 --- /dev/null +++ b/tajo-storage/tajo-storage-common/pom.xml @@ -0,0 +1,333 @@ + + + + + + tajo-project + org.apache.tajo + 0.9.1-SNAPSHOT + ../../tajo-project + + 4.0.0 + tajo-storage-common + jar + Tajo Storage Common + + UTF-8 + UTF-8 + + + + + repository.jboss.org + https://repository.jboss.org/nexus/content/repositories/releases/ + + + false + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + ${project.build.sourceEncoding} + + + + org.apache.rat + apache-rat-plugin + + + verify + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + TRUE + + -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../../tajo-common/src/main/proto + --proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/IndexProtos.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 2.7.1 + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + + + + org.apache.tajo + tajo-common + + + org.apache.tajo + tajo-catalog-common + + + org.apache.tajo + tajo-plan + + + + org.apache.hadoop + hadoop-common + provided + + + zookeeper + org.apache.zookeeper + + + slf4j-api + org.slf4j + + + jersey-json + com.sun.jersey + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + + + org.apache.hadoop + hadoop-minicluster + test + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + hadoop-yarn-server-tests + org.apache.hadoop + + + hadoop-mapreduce-client-jobclient + org.apache.hadoop + + + hadoop-mapreduce-client-app + org.apache.hadoop + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-mapreduce-client-hs + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + com.google.protobuf + protobuf-java + + + junit + junit + test + + + + + + docs + + false + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + module-javadocs + package + + jar + + + ${project.build.directory} + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + 2.15 + + + + \ No newline at end of file diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Appender.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Appender.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/DataLocation.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DataLocation.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/DataLocation.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DataLocation.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/DiskInfo.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskInfo.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/DiskInfo.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskInfo.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/DiskMountInfo.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskMountInfo.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/DiskMountInfo.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskMountInfo.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/DiskUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskUtil.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/DiskUtil.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskUtil.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FrameTuple.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/FrameTuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/FrameTuple.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/FrameTuple.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/LazyTuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/LazyTuple.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MemoryUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MemoryUtil.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/MemoryUtil.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MemoryUtil.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/NullScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java similarity index 59% rename from tajo-storage/src/main/java/org/apache/tajo/storage/NullScanner.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java index 19d333e066..42722288a8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/NullScanner.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java @@ -17,21 +17,43 @@ */ import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; -public class NullScanner extends FileScanner { +public class NullScanner implements Scanner { + protected final Configuration conf; + protected final TableMeta meta; + protected final Schema schema; + protected final Fragment fragment; + protected final int columnNum; + protected Column [] targets; + protected float progress; + protected TableStats tableStats; + public NullScanner(Configuration conf, Schema schema, TableMeta meta, Fragment fragment) { - super(conf, schema, meta, fragment); + this.conf = conf; + this.meta = meta; + this.schema = schema; + this.fragment = fragment; + this.tableStats = new TableStats(); + this.columnNum = this.schema.size(); + } + + @Override + public void init() throws IOException { + progress = 0.0f; + tableStats.setNumBytes(0); + tableStats.setNumBlocks(0); } @Override public Tuple next() throws IOException { progress = 1.0f; - return null; } @@ -42,7 +64,7 @@ public void reset() throws IOException { @Override public void close() throws IOException { - progress = 0.0f; + progress = 1.0f; } @Override @@ -50,13 +72,38 @@ public boolean isProjectable() { return false; } + @Override + public void setTarget(Column[] targets) { + this.targets = targets; + } + @Override public boolean isSelectable() { return true; } + @Override + public void setSearchCondition(Object expr) { + + } + @Override public boolean isSplittable() { return true; } + + @Override + public float getProgress() { + return progress; + } + + @Override + public TableStats getInputStats() { + return tableStats; + } + + @Override + public Schema getSchema() { + return schema; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/NumericPathComparator.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NumericPathComparator.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/NumericPathComparator.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NumericPathComparator.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/Scanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java similarity index 87% rename from tajo-storage/src/main/java/org/apache/tajo/storage/Scanner.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java index 16c4faa4a5..0356b193d2 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/Scanner.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java @@ -38,22 +38,22 @@ public interface Scanner extends SchemaObject, Closeable { * @return retrieve null if the scanner has no more tuples. * Otherwise it returns one tuple. * - * @throws IOException if internal I/O error occurs during next method + * @throws java.io.IOException if internal I/O error occurs during next method */ Tuple next() throws IOException; - + /** - * Reset the cursor. After executed, the scanner + * Reset the cursor. After executed, the scanner * will retrieve the first tuple. - * - * @throws IOException if internal I/O error occurs during reset method + * + * @throws java.io.IOException if internal I/O error occurs during reset method */ void reset() throws IOException; - + /** * Close scanner - * - * @throws IOException if internal I/O error occurs during close method + * + * @throws java.io.IOException if internal I/O error occurs during close method */ void close() throws IOException; @@ -69,7 +69,7 @@ public interface Scanner extends SchemaObject, Closeable { * Set target columns * @param targets columns to be projected */ - void setTarget(Column [] targets); + void setTarget(Column[] targets); /** * It returns if the selection is executed in the underlying scanner layer. diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/SeekableScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/SeekableScanner.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/SeekableScanner.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/SeekableScanner.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/SerializerDeserializer.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/SerializerDeserializer.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/SerializerDeserializer.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/SerializerDeserializer.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java similarity index 91% rename from tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java index 1e81816b29..e14b20a6f2 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -22,28 +22,35 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.tajo.*; -import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.plan.rewrite.RewriteRule; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; -import org.apache.tajo.storage.hbase.HBaseStorageManager; import org.apache.tajo.util.TUtil; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.URI; import java.text.NumberFormat; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** @@ -104,7 +111,7 @@ public StorageManager(StoreType storeType) { /** * Initialize storage manager. - * @throws IOException + * @throws java.io.IOException */ protected abstract void storageInit() throws IOException; @@ -114,7 +121,7 @@ public StorageManager(StoreType storeType) { * * @param tableDesc Table description which is created. * @param ifNotExists Creates the table only when the table does not exist. - * @throws IOException + * @throws java.io.IOException */ public abstract void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException; @@ -123,7 +130,7 @@ public StorageManager(StoreType storeType) { * which is the option to delete all the data. * * @param tableDesc - * @throws IOException + * @throws java.io.IOException */ public abstract void purgeTable(TableDesc tableDesc) throws IOException; @@ -134,7 +141,7 @@ public StorageManager(StoreType storeType) { * @param tableDesc The table description for the target data. * @param scanNode The logical node for scanning. * @return The list of input fragments. - * @throws IOException + * @throws java.io.IOException */ public abstract List getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException; @@ -146,7 +153,7 @@ public abstract List getSplits(String fragmentId, TableDesc tableDesc, * @param currentPage The current page number within the entire list. * @param numFragments The number of fragments in the result. * @return The list of input fragments. - * @throws IOException + * @throws java.io.IOException */ public abstract List getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException; @@ -172,7 +179,7 @@ public abstract List getNonForwardSplit(TableDesc tableDesc, int curre * @param inputSchema The input schema * @param sortSpecs The sort specification that contains the sort column and sort order. * @return The list of sort ranges. - * @throws IOException + * @throws java.io.IOException */ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, @@ -185,7 +192,7 @@ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, T * That kind of the storage should implements the logic related to creating table in this method. * * @param node The child node of the root node. - * @throws IOException + * @throws java.io.IOException */ public abstract void beforeInsertOrCATS(LogicalNode node) throws IOException; @@ -194,7 +201,7 @@ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, T * Each storage manager should implement to be processed when the query fails in this method. * * @param node The child node of the root node. - * @throws IOException + * @throws java.io.IOException */ public abstract void queryFailed(LogicalNode node) throws IOException; @@ -210,7 +217,7 @@ public StoreType getStoreType() { * Initialize StorageManager instance. It should be called before using. * * @param tajoConf - * @throws IOException + * @throws java.io.IOException */ public void init(TajoConf tajoConf) throws IOException { this.conf = tajoConf; @@ -219,7 +226,7 @@ public void init(TajoConf tajoConf) throws IOException { /** * Close StorageManager - * @throws IOException + * @throws java.io.IOException */ public void close() throws IOException { synchronized(storageManagers) { @@ -236,7 +243,7 @@ public void close() throws IOException { * @param fragmentId The table name or previous ExecutionBlockId * @param tableDesc The table description for the target data. * @return The list of input fragments. - * @throws IOException + * @throws java.io.IOException */ public List getSplits(String fragmentId, TableDesc tableDesc) throws IOException { return getSplits(fragmentId, tableDesc, null); @@ -247,9 +254,9 @@ public List getSplits(String fragmentId, TableDesc tableDesc) throws I * * @param tajoConf Tajo system property. * @return - * @throws IOException + * @throws java.io.IOException */ - public static FileStorageManager getFileStorageManager(TajoConf tajoConf) throws IOException { + public static StorageManager getFileStorageManager(TajoConf tajoConf) throws IOException { return getFileStorageManager(tajoConf, null); } @@ -259,9 +266,9 @@ public static FileStorageManager getFileStorageManager(TajoConf tajoConf) throws * @param tajoConf Tajo system property. * @param warehousePath The warehouse directory to be set in the tajoConf. * @return - * @throws IOException + * @throws java.io.IOException */ - public static FileStorageManager getFileStorageManager(TajoConf tajoConf, Path warehousePath) throws IOException { + public static StorageManager getFileStorageManager(TajoConf tajoConf, Path warehousePath) throws IOException { URI uri; TajoConf copiedConf = new TajoConf(tajoConf); if (warehousePath != null) { @@ -269,7 +276,7 @@ public static FileStorageManager getFileStorageManager(TajoConf tajoConf, Path w } uri = TajoConf.getWarehouseDir(copiedConf).toUri(); String key = "file".equals(uri.getScheme()) ? "file" : uri.toString(); - return (FileStorageManager) getStorageManager(copiedConf, StoreType.CSV, key); + return getStorageManager(copiedConf, StoreType.CSV, key); } /** @@ -278,7 +285,7 @@ public static FileStorageManager getFileStorageManager(TajoConf tajoConf, Path w * @param tajoConf Tajo system property. * @param storeType Storage type * @return - * @throws IOException + * @throws java.io.IOException */ public static StorageManager getStorageManager(TajoConf tajoConf, String storeType) throws IOException { if ("HBASE".equals(storeType)) { @@ -294,7 +301,7 @@ public static StorageManager getStorageManager(TajoConf tajoConf, String storeTy * @param tajoConf Tajo system property. * @param storeType Storage type * @return - * @throws IOException + * @throws java.io.IOException */ public static StorageManager getStorageManager(TajoConf tajoConf, StoreType storeType) throws IOException { return getStorageManager(tajoConf, storeType, null); @@ -307,7 +314,7 @@ public static StorageManager getStorageManager(TajoConf tajoConf, StoreType stor * @param storeType Storage type * @param managerKey Key that can identify each storage manager(may be a path) * @return - * @throws IOException + * @throws java.io.IOException */ public static synchronized StorageManager getStorageManager ( TajoConf tajoConf, StoreType storeType, String managerKey) throws IOException { @@ -315,14 +322,35 @@ public static synchronized StorageManager getStorageManager ( String storeKey = storeType + managerKey; StorageManager manager = storageManagers.get(storeKey); if (manager == null) { + String typeName = "hdfs"; + switch (storeType) { case HBASE: - manager = new HBaseStorageManager(storeType); + typeName = "hbase"; break; default: - manager = new FileStorageManager(storeType); + typeName = "hdfs"; } + Class storageManagerClass = + tajoConf.getClass(String.format("tajo.storage.manager.%s.class", typeName), null, StorageManager.class); + + if (storageManagerClass == null) { + throw new IOException("Unknown Storage Type: " + typeName); + } + + try { + Constructor constructor = + (Constructor) CONSTRUCTOR_CACHE.get(storageManagerClass); + if (constructor == null) { + constructor = storageManagerClass.getDeclaredConstructor(new Class[]{StoreType.class}); + constructor.setAccessible(true); + CONSTRUCTOR_CACHE.put(storageManagerClass, constructor); + } + manager = constructor.newInstance(new Object[]{storeType}); + } catch (Exception e) { + throw new RuntimeException(e); + } manager.init(tajoConf); storageManagers.put(storeKey, manager); } @@ -339,7 +367,7 @@ public static synchronized StorageManager getStorageManager ( * @param fragment The fragment for scanning * @param target Columns which are selected. * @return Scanner instance - * @throws IOException + * @throws java.io.IOException */ public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException { return getScanner(meta, schema, FragmentConvertor.convert(conf, fragment), target); @@ -352,7 +380,7 @@ public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, * @param schema The input schema * @param fragment The fragment for scanning * @return Scanner instance - * @throws IOException + * @throws java.io.IOException */ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException { return getScanner(meta, schema, fragment, schema); @@ -366,7 +394,7 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) thro * @param fragment The fragment for scanning * @param target The output schema * @return Scanner instance - * @throws IOException + * @throws java.io.IOException */ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { if (fragment.isEmpty()) { @@ -396,33 +424,13 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Sche * @param fragment The fragment for scanning * @param target The output schema * @return Scanner instance - * @throws IOException + * @throws java.io.IOException */ public static synchronized SeekableScanner getSeekableScanner( - TajoConf conf, TableMeta meta, Schema schema, FileFragment fragment, Schema target) throws IOException { + TajoConf conf, TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { return (SeekableScanner)getStorageManager(conf, meta.getStoreType()).getScanner(meta, schema, fragment, target); } - /** - * Returns Scanner instance. - * - * @param conf The system property - * @param meta The table meta - * @param schema The input schema - * @param path The data file path - * @return Scanner instance - * @throws IOException - */ - public static synchronized SeekableScanner getSeekableScanner( - TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { - - FileSystem fs = path.getFileSystem(conf); - FileStatus status = fs.getFileStatus(path); - FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); - - return getSeekableScanner(conf, meta, schema, fragment, schema); - } - /** * Returns Appender instance. * @param queryContext Query property. @@ -431,7 +439,7 @@ public static synchronized SeekableScanner getSeekableScanner( * @param schema Output schema. * @param workDir Working directory * @return Appender instance - * @throws IOException + * @throws java.io.IOException */ public Appender getAppender(OverridableConf queryContext, QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir) @@ -522,7 +530,7 @@ public static T newAppenderInstance(Class theClass, Configuration conf, Q * * @param storeType store type * @return The Scanner class - * @throws IOException + * @throws java.io.IOException */ public Class getScannerClass(CatalogProtos.StoreType storeType) throws IOException { String handlerName = storeType.name().toLowerCase(); @@ -561,7 +569,7 @@ public static long getFragmentLength(TajoConf conf, Fragment fragment) { * * @param tableDesc The table description of insert target. * @param outSchema The output schema of select query for inserting. - * @throws IOException + * @throws java.io.IOException */ public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { // nothing to do @@ -574,7 +582,7 @@ public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throw * @param queryContext The query property * @param tableDesc The description of the target table. * @return The list of storage specified rewrite rules - * @throws IOException + * @throws java.io.IOException */ public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { return null; @@ -591,7 +599,7 @@ public List getRewriteRules(OverridableConf queryContext, TableDesc * @param schema The final output schema * @param tableDesc The description of the target table * @return Saved path - * @throws IOException + * @throws java.io.IOException */ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, @@ -611,7 +619,7 @@ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId fina * @param tableDesc The description of the target table * @param changeFileSeq If true change result file name with max sequence. * @return Saved path - * @throws IOException + * @throws java.io.IOException */ protected Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, @@ -751,7 +759,7 @@ protected Path commitOutputData(OverridableConf queryContext, ExecutionBlockId f * @param finalOutputPath Final output path * @param nf Number format * @param fileSeq The sequence number - * @throws IOException + * @throws java.io.IOException */ private void moveResultFromStageToFinal(FileSystem fs, Path stagingResultDir, FileStatus fileStatus, Path finalOutputPath, @@ -828,7 +836,7 @@ private String extractSubPath(Path parentPath, Path childPath) { * @param seq sequence number * @param nf Number format * @return New path attached with sequence number - * @throws IOException + * @throws java.io.IOException */ private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IOException { String[] tokens = path.getName().split("-"); @@ -843,7 +851,7 @@ private String replaceFileNameSeq(Path path, int seq, NumberFormat nf) throws IO * @param fs FileSystem * @param stagingPath The stagind directory * @return - * @throws IOException + * @throws java.io.IOException */ private boolean verifyAllFileMoved(FileSystem fs, Path stagingPath) throws IOException { FileStatus[] files = fs.listStatus(stagingPath); @@ -874,7 +882,7 @@ private boolean verifyAllFileMoved(FileSystem fs, Path stagingPath) throws IOExc * @param stagingPath * @param outputPath * @param stagingParentPathString - * @throws IOException + * @throws java.io.IOException */ private void visitPartitionedDirectory(FileSystem fs, Path stagingPath, Path outputPath, String stagingParentPathString, diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageProperty.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/StorageProperty.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java similarity index 83% rename from tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java index 4a666783c0..54fdb69313 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java @@ -26,11 +26,8 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.KeyValueSet; -import parquet.hadoop.ParquetOutputFormat; import sun.nio.ch.DirectBuffer; import java.io.DataInput; @@ -112,26 +109,6 @@ public static Path concatPath(Path parent, String...childs) { return new Path(parent, sb.toString()); } - public static KeyValueSet newPhysicalProperties(CatalogProtos.StoreType type) { - KeyValueSet options = new KeyValueSet(); - if (CatalogProtos.StoreType.CSV == type) { - options.set(CSVFILE_DELIMITER, DEFAULT_FIELD_DELIMITER); - } else if (CatalogProtos.StoreType.RCFILE == type) { - options.set(RCFILE_SERDE, DEFAULT_BINARY_SERDE); - } else if (CatalogProtos.StoreType.SEQUENCEFILE == type) { - options.set(SEQUENCEFILE_SERDE, DEFAULT_TEXT_SERDE); - options.set(SEQUENCEFILE_DELIMITER, DEFAULT_FIELD_DELIMITER); - } else if (type == CatalogProtos.StoreType.PARQUET) { - options.set(ParquetOutputFormat.BLOCK_SIZE, PARQUET_DEFAULT_BLOCK_SIZE); - options.set(ParquetOutputFormat.PAGE_SIZE, PARQUET_DEFAULT_PAGE_SIZE); - options.set(ParquetOutputFormat.COMPRESSION, PARQUET_DEFAULT_COMPRESSION_CODEC_NAME); - options.set(ParquetOutputFormat.ENABLE_DICTIONARY, PARQUET_DEFAULT_IS_DICTIONARY_ENABLED); - options.set(ParquetOutputFormat.VALIDATION, PARQUET_DEFAULT_IS_VALIDATION_ENABLED); - } - - return options; - } - static final String fileNamePatternV08 = "part-[0-9]*-[0-9]*"; static final String fileNamePatternV09 = "part-[0-9]*-[0-9]*-[0-9]*"; @@ -241,8 +218,4 @@ public static void skipFully(DataInput in, int len) throws IOException { amt -= ret; } } - - public static boolean isFileStorageType(StoreType storageType) { - return true; - } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableStatistics.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableStatistics.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleComparator.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/TupleComparator.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleComparator.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TupleRange.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/TupleRange.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/compress/CodecPool.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/compress/CodecPool.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/compress/CodecPool.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/compress/CodecPool.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/exception/AlreadyExistsStorageException.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/AlreadyExistsStorageException.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/exception/AlreadyExistsStorageException.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/AlreadyExistsStorageException.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/exception/UnknownCodecException.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/UnknownCodecException.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/exception/UnknownCodecException.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/UnknownCodecException.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/exception/UnknownDataTypeException.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/UnknownDataTypeException.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/exception/UnknownDataTypeException.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/UnknownDataTypeException.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/exception/UnsupportedFileTypeException.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/UnsupportedFileTypeException.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/exception/UnsupportedFileTypeException.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/exception/UnsupportedFileTypeException.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/Fragment.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/Fragment.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/fragment/Fragment.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/Fragment.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/BaseTupleBuilder.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/RowBlockReader.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/RowBlockReader.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/RowBlockReader.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/TupleBuilder.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/TupleBuilder.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/TupleBuilder.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/DirectBufTuple.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/FixedSizeLimitSpec.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapMemory.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlock.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockReader.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockUtils.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowBlockWriter.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/OffHeapRowWriter.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ResizableLimitSpec.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/RowWriter.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTupleBytesComparator.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/ZeroCopyTuple.java diff --git a/tajo-storage/src/main/proto/IndexProtos.proto b/tajo-storage/tajo-storage-common/src/main/proto/IndexProtos.proto similarity index 100% rename from tajo-storage/src/main/proto/IndexProtos.proto rename to tajo-storage/tajo-storage-common/src/main/proto/IndexProtos.proto diff --git a/tajo-storage/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml similarity index 94% rename from tajo-storage/src/main/resources/storage-default.xml rename to tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index efe7dd288f..43921c1810 100644 --- a/tajo-storage/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -20,6 +20,16 @@ --> + + + tajo.storage.manager.hdfs.class + org.apache.tajo.storage.FileStorageManager + + + tajo.storage.manager.hbase.class + org.apache.tajo.storage.hbase.HBaseStorageManager + + tajo.storage.scanner-handler diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestFrameTuple.java b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestFrameTuple.java similarity index 100% rename from tajo-storage/src/test/java/org/apache/tajo/storage/TestFrameTuple.java rename to tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestFrameTuple.java index 387fed550d..0251dc7d6a 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestFrameTuple.java +++ b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestFrameTuple.java @@ -18,11 +18,11 @@ package org.apache.tajo.storage; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestLazyTuple.java similarity index 100% rename from tajo-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java rename to tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestLazyTuple.java diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java similarity index 100% rename from tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java rename to tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestVTuple.java similarity index 99% rename from tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java rename to tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestVTuple.java index 9837fd1887..1bbd9ec230 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java +++ b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestVTuple.java @@ -19,16 +19,16 @@ package org.apache.tajo.storage; +import org.apache.tajo.datum.DatumFactory; import org.junit.Before; import org.junit.Test; -import org.apache.tajo.datum.DatumFactory; import static org.junit.Assert.*; public class TestVTuple { /** - * @throws java.lang.Exception + * @throws Exception */ @Before public void setUp() throws Exception { diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml new file mode 100644 index 0000000000..d1c561b708 --- /dev/null +++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml @@ -0,0 +1,164 @@ + + + + + + + + fs.s3.impl + org.apache.tajo.storage.s3.SmallBlockS3FileSystem + + + + + tajo.storage.manager.hdfs.class + org.apache.tajo.storage.FileStorageManager + + + tajo.storage.manager.hbase.class + org.apache.tajo.storage.hbase.HBaseStorageManager + + + + + tajo.storage.scanner-handler + csv,raw,rcfile,row,trevni,parquet,sequencefile,avro + + + + + tajo.storage.fragment.csv.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.raw.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.rcfile.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.row.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.trevni.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.parquet.class + org.apache.tajo.storage.FileFragment + + + tajo.storage.fragment.sequencefile.class + org.apache.tajo.storage.fragment.FileFragment + + + tajo.storage.fragment.avro.class + org.apache.tajo.storage.fragment.FileFragment + + + + + tajo.storage.scanner-handler.csv.class + org.apache.tajo.storage.CSVFile$CSVScanner + + + + tajo.storage.scanner-handler.raw.class + org.apache.tajo.storage.RawFile$RawFileScanner + + + + tajo.storage.scanner-handler.rcfile.class + org.apache.tajo.storage.rcfile.RCFile$RCFileScanner + + + + tajo.storage.scanner-handler.rowfile.class + org.apache.tajo.storage.RowFile$RowFileScanner + + + + tajo.storage.scanner-handler.trevni.class + org.apache.tajo.storage.trevni.TrevniScanner + + + + tajo.storage.scanner-handler.parquet.class + org.apache.tajo.storage.parquet.ParquetScanner + + + + tajo.storage.scanner-handler.sequencefile.class + org.apache.tajo.storage.sequencefile.SequenceFileScanner + + + + tajo.storage.scanner-handler.avro.class + org.apache.tajo.storage.avro.AvroScanner + + + + + tajo.storage.appender-handler + csv,raw,rcfile,row,trevni,parquet,sequencefile,avro + + + + tajo.storage.appender-handler.csv.class + org.apache.tajo.storage.CSVFile$CSVAppender + + + + tajo.storage.appender-handler.raw.class + org.apache.tajo.storage.RawFile$RawFileAppender + + + + tajo.storage.appender-handler.rcfile.class + org.apache.tajo.storage.rcfile.RCFile$RCFileAppender + + + + tajo.storage.appender-handler.rowfile.class + org.apache.tajo.storage.RowFile$RowFileAppender + + + + tajo.storage.appender-handler.trevni.class + org.apache.tajo.storage.trevni.TrevniAppender + + + + tajo.storage.appender-handler.parquet.class + org.apache.tajo.storage.parquet.ParquetAppender + + + + tajo.storage.appender-handler.sequencefile.class + org.apache.tajo.storage.sequencefile.SequenceFileAppender + + + + tajo.storage.appender-handler.avro.class + org.apache.tajo.storage.avro.AvroAppender + + diff --git a/tajo-storage/tajo-storage-hbase/pom.xml b/tajo-storage/tajo-storage-hbase/pom.xml new file mode 100644 index 0000000000..e37149da4d --- /dev/null +++ b/tajo-storage/tajo-storage-hbase/pom.xml @@ -0,0 +1,349 @@ + + + + + + tajo-project + org.apache.tajo + 0.9.1-SNAPSHOT + ../../tajo-project + + 4.0.0 + + tajo-storage-hbase + jar + Tajo HBase Storage + + UTF-8 + UTF-8 + + + + + repository.jboss.org + https://repository.jboss.org/nexus/content/repositories/releases/ + + + false + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + ${project.build.sourceEncoding} + + + + org.apache.rat + apache-rat-plugin + + + verify + + check + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + TRUE + + -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../../tajo-common/src/main/proto + --proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/StorageFragmentProtos.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + + + + org.apache.tajo + tajo-common + provided + + + org.apache.tajo + tajo-catalog-common + provided + + + org.apache.tajo + tajo-plan + provided + + + org.apache.tajo + tajo-storage-common + provided + + + + org.apache.hadoop + hadoop-common + provided + + + zookeeper + org.apache.zookeeper + + + slf4j-api + org.slf4j + + + jersey-json + com.sun.jersey + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + + + org.apache.hadoop + hadoop-minicluster + test + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + hadoop-yarn-server-tests + org.apache.hadoop + + + hadoop-mapreduce-client-jobclient + org.apache.hadoop + + + hadoop-mapreduce-client-app + org.apache.hadoop + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-mapreduce-client-hs + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + com.google.protobuf + protobuf-java + + + junit + junit + test + + + org.apache.hbase + hbase-server + ${hbase.version} + provided + + + org.apache.hbase + hbase-client + ${hbase.version} + provided + + + + + + docs + + false + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + module-javadocs + package + + jar + + + ${project.build.directory} + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + 2.15 + + + + \ No newline at end of file diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java similarity index 94% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java index 8aefebba73..5646793127 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java @@ -25,8 +25,11 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; -import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalRootNode; +import org.apache.tajo.plan.logical.SortNode; import org.apache.tajo.plan.logical.SortNode.SortRangeType; +import org.apache.tajo.plan.logical.UnaryNode; import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.plan.util.PlannerUtil; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java index 43ad7f3aed..2674511a74 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -26,7 +26,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.fragment.StorageFragmentProtos.HBaseFragmentProto; +import org.apache.tajo.storage.hbase.StorageFragmentProtos.*; public class HBaseFragment implements Fragment, Comparable, Cloneable { @Expose diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index c011eadda8..0bc944ea7b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -22,7 +22,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index 5cdbe444e4..c9630823d8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -25,7 +25,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; @@ -42,11 +45,17 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.expr.*; -import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.logical.CreateTableNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.plan.rewrite.RewriteRule; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.util.*; +import org.apache.tajo.util.Bytes; +import org.apache.tajo.util.BytesUtils; +import org.apache.tajo.util.Pair; +import org.apache.tajo.util.TUtil; import java.io.BufferedReader; import java.io.IOException; @@ -196,7 +205,7 @@ private void createTable(TableMeta tableMeta, Schema schema, * @param schema * @param meta * @return - * @throws IOException + * @throws java.io.IOException */ private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) throws IOException { String splitRowKeys = meta.getOption(META_SPLIT_ROW_KEYS_KEY, ""); @@ -293,7 +302,7 @@ private byte[][] getSplitKeys(TajoConf conf, Schema schema, TableMeta meta) thro * @param conf * @param tableMeta * @return - * @throws IOException + * @throws java.io.IOException */ public static Configuration getHBaseConfiguration(Configuration conf, TableMeta tableMeta) throws IOException { String zkQuorum = tableMeta.getOption(META_ZK_QUORUM_KEY, ""); @@ -319,7 +328,7 @@ public static Configuration getHBaseConfiguration(Configuration conf, TableMeta * @param tableMeta * @param schema * @return - * @throws IOException + * @throws java.io.IOException */ public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema schema) throws IOException { String hbaseTableName = tableMeta.getOption(META_TABLE_KEY, ""); @@ -366,7 +375,7 @@ public void purgeTable(TableDesc tableDesc) throws IOException { * * @param tableDesc * @return - * @throws IOException + * @throws java.io.IOException */ private Column[] getIndexableColumns(TableDesc tableDesc) throws IOException { ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTextSerializerDeserializer.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java index b9425f9fad..07f7988d2a 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java @@ -22,7 +22,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -31,6 +30,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.tajo.ExecutionBlockId; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/IndexPredication.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/RowKeyMapping.java diff --git a/tajo-storage/src/main/proto/StorageFragmentProtos.proto b/tajo-storage/tajo-storage-hbase/src/main/proto/StorageFragmentProtos.proto similarity index 95% rename from tajo-storage/src/main/proto/StorageFragmentProtos.proto rename to tajo-storage/tajo-storage-hbase/src/main/proto/StorageFragmentProtos.proto index dd79d74b6e..668b1161a6 100644 --- a/tajo-storage/src/main/proto/StorageFragmentProtos.proto +++ b/tajo-storage/tajo-storage-hbase/src/main/proto/StorageFragmentProtos.proto @@ -16,7 +16,7 @@ * limitations under the License. */ -option java_package = "org.apache.tajo.storage.fragment"; +option java_package = "org.apache.tajo.storage.hbase"; option java_outer_classname = "StorageFragmentProtos"; option optimize_for = SPEED; option java_generic_services = false; diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java similarity index 92% rename from tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java rename to tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java index f242b118bf..3f0dcd0f26 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java @@ -18,20 +18,16 @@ package org.apache.tajo.storage.hbase; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.util.Bytes; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class TestColumnMapping { @Test diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java similarity index 100% rename from tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java rename to tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java diff --git a/tajo-storage/tajo-storage-hdfs/pom.xml b/tajo-storage/tajo-storage-hdfs/pom.xml new file mode 100644 index 0000000000..de6d0aed7d --- /dev/null +++ b/tajo-storage/tajo-storage-hdfs/pom.xml @@ -0,0 +1,398 @@ + + + + + + tajo-project + org.apache.tajo + 0.9.1-SNAPSHOT + ../../tajo-project + + 4.0.0 + + tajo-storage-hdfs + jar + Tajo HDFS Storage + + UTF-8 + UTF-8 + 1.5.0 + 2.1.0 + + + + + repository.jboss.org + https://repository.jboss.org/nexus/content/repositories/releases/ + + + false + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + ${project.build.sourceEncoding} + + + + org.apache.rat + apache-rat-plugin + + + verify + + check + + + + + + src/test/resources/testVariousTypes.avsc + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + TRUE + + -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../../tajo-common/src/main/proto + --proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/StorageFragmentProtos.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + + + + org.apache.tajo + tajo-common + provided + + + org.apache.tajo + tajo-catalog-common + provided + + + org.apache.tajo + tajo-plan + provided + + + org.apache.tajo + tajo-storage-common + provided + + + + org.apache.avro + trevni-core + 1.7.3 + + + org.apache.avro + trevni-avro + 1.7.3 + + + org.apache.hadoop + hadoop-core + + + + + org.apache.hadoop + hadoop-common + provided + + + zookeeper + org.apache.zookeeper + + + slf4j-api + org.slf4j + + + jersey-json + com.sun.jersey + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + + + org.apache.hadoop + hadoop-minicluster + test + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + hadoop-yarn-server-tests + org.apache.hadoop + + + hadoop-mapreduce-client-jobclient + org.apache.hadoop + + + hadoop-mapreduce-client-app + org.apache.hadoop + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-mapreduce-client-hs + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + com.google.protobuf + protobuf-java + provided + + + junit + junit + test + + + com.twitter + parquet-column + ${parquet.version} + + + com.twitter + parquet-hadoop + ${parquet.version} + + + com.twitter + parquet-format + ${parquet.format.version} + + + + + + docs + + false + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + module-javadocs + package + + jar + + + ${project.build.directory} + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + 2.15 + + + + \ No newline at end of file diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java index 04bcf71947..b4a550d84b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java @@ -24,7 +24,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.*; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java similarity index 94% rename from tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java index 117d3da736..f35b4e824f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java @@ -53,7 +53,8 @@ public FileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema try { if (taskAttemptId != null) { - this.path = StorageManager.getFileStorageManager((TajoConf) conf).getAppenderFilePath(taskAttemptId, workDir); + this.path = ((FileStorageManager)StorageManager.getFileStorageManager((TajoConf) conf)) + .getAppenderFilePath(taskAttemptId, workDir); } else { this.path = workDir; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/FileScanner.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java similarity index 96% rename from tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java index d382f41fc9..ce438be65a 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -27,7 +27,9 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.tajo.*; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.QueryUnitAttemptId; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; @@ -36,7 +38,7 @@ import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.util.*; +import org.apache.tajo.util.Bytes; import java.io.IOException; import java.text.NumberFormat; @@ -133,7 +135,7 @@ public boolean exists(Path path) throws IOException { * This method deletes only data contained in the given path. * * @param path The path in which data are deleted. - * @throws IOException + * @throws java.io.IOException */ public void deleteData(Path path) throws IOException { FileSystem fileSystem = path.getFileSystem(conf); @@ -319,7 +321,7 @@ public boolean accept(Path path) { * expression. * * @return array of FileStatus objects - * @throws IOException if zero items. + * @throws java.io.IOException if zero items. */ protected List listStatus(Path... dirs) throws IOException { List result = new ArrayList(); @@ -506,7 +508,7 @@ private Map> getVolumeMap(List frags) { /** * Generate the list of files and make them into FileSplits. * - * @throws IOException + * @throws java.io.IOException */ public List getSplits(String tableName, TableMeta meta, Schema schema, Path... inputs) throws IOException { @@ -828,4 +830,24 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc throws IOException { return null; } + + /** + * Returns Scanner instance. + * + * @param conf The system property + * @param meta The table meta + * @param schema The input schema + * @param path The data file path + * @return Scanner instance + * @throws java.io.IOException + */ + public static synchronized SeekableScanner getSeekableScanner( + TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { + + FileSystem fs = path.getFileSystem(conf); + FileStatus status = fs.getFileStatus(path); + FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); + + return getSeekableScanner(conf, meta, schema, fragment, schema); + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java index 40cad323ec..33b2750035 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java @@ -79,7 +79,7 @@ public void init() throws IOException { * @param taskId * @param tuples * @return written bytes - * @throws IOException + * @throws java.io.IOException */ public int addTuples(QueryUnitAttemptId taskId, List tuples) throws IOException { synchronized(appender) { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java similarity index 97% rename from tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index 33a9233dfd..636ae0f4d7 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -84,8 +84,8 @@ public HashShuffleAppender getAppender(TajoConf tajoConf, ExecutionBlockId ebId, if (!fs.exists(dataFile.getParent())) { fs.mkdirs(dataFile.getParent()); } - FileAppender appender = (FileAppender) StorageManager.getFileStorageManager( - tajoConf, null).getAppender(meta, outSchema, dataFile); + FileAppender appender = (FileAppender)((FileStorageManager)StorageManager.getFileStorageManager( + tajoConf, null)).getAppender(meta, outSchema, dataFile); appender.enableStats(); appender.init(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/LineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/LineReader.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/LineReader.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/LineReader.java index 66c610a16d..0f31bafb27 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/LineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/LineReader.java @@ -60,7 +60,7 @@ public class LineReader implements Closeable { * default buffer-size (64k). * * @param in The input stream - * @throws IOException + * @throws java.io.IOException */ public LineReader(InputStream in) { this(in, DEFAULT_BUFFER_SIZE); @@ -72,7 +72,7 @@ public LineReader(InputStream in) { * * @param in The input stream * @param bufferSize Size of the read buffer - * @throws IOException + * @throws java.io.IOException */ public LineReader(InputStream in, int bufferSize) { this.in = in; @@ -88,7 +88,7 @@ public LineReader(InputStream in, int bufferSize) { * * @param in input stream * @param conf configuration - * @throws IOException + * @throws java.io.IOException */ public LineReader(InputStream in, Configuration conf) throws IOException { this(in, conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE)); @@ -117,7 +117,7 @@ public LineReader(InputStream in, byte[] recordDelimiterBytes) { * @param in The input stream * @param bufferSize Size of the read buffer * @param recordDelimiterBytes The delimiter - * @throws IOException + * @throws java.io.IOException */ public LineReader(InputStream in, int bufferSize, byte[] recordDelimiterBytes) { @@ -136,7 +136,7 @@ public LineReader(InputStream in, int bufferSize, * @param in input stream * @param conf configuration * @param recordDelimiterBytes The delimiter - * @throws IOException + * @throws java.io.IOException */ public LineReader(InputStream in, Configuration conf, byte[] recordDelimiterBytes) throws IOException { @@ -150,7 +150,7 @@ public LineReader(InputStream in, Configuration conf, /** * Close the underlying stream. * - * @throws IOException + * @throws java.io.IOException */ public void close() throws IOException { in.close(); @@ -174,7 +174,7 @@ public void reset() { * potentially by as much as one buffer length. * @return the number of bytes read including the (longest) newline * found. - * @throws IOException if the underlying stream throws + * @throws java.io.IOException if the underlying stream throws */ public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { @@ -540,7 +540,7 @@ private int readCustomLine(Text str, int maxLineLength, int maxBytesToConsume) * @param str the object to store the given line * @param maxLineLength the maximum number of bytes to store into str. * @return the number of bytes read including the newline - * @throws IOException if the underlying stream throws + * @throws java.io.IOException if the underlying stream throws */ public int readLine(Text str, int maxLineLength) throws IOException { return readLine(str, maxLineLength, Integer.MAX_VALUE); @@ -551,7 +551,7 @@ public int readLine(Text str, int maxLineLength) throws IOException { * * @param str the object to store the given line * @return the number of bytes read including the newline - * @throws IOException if the underlying stream throws + * @throws java.io.IOException if the underlying stream throws */ public int readLine(Text str) throws IOException { return readLine(str, Integer.MAX_VALUE, Integer.MAX_VALUE); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java index 003f8e5b44..c0d30d06fa 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java @@ -19,7 +19,6 @@ package org.apache.tajo.storage; import com.google.protobuf.Message; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java index efb23eb71c..8da6ada4ed 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java @@ -128,7 +128,7 @@ private void readHeader() throws IOException { * Find the sync from the front of the buffer * * @return return true if it succeeds to find the sync. - * @throws IOException + * @throws java.io.IOException */ private boolean checkSync() throws IOException { buffer.getInt(); // escape diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/SplitLineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/SplitLineReader.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/SplitLineReader.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/SplitLineReader.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java index 9e1e7ea8aa..85f91cc916 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java @@ -18,31 +18,29 @@ package org.apache.tajo.storage.avro; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; -import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.DatumWriter; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.storage.FileAppender; import org.apache.tajo.storage.TableStatistics; import org.apache.tajo.storage.Tuple; +import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; /** * FileAppender for writing to Avro files. diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java index 72472fced2..51594df5d1 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java @@ -38,7 +38,6 @@ import org.apache.tajo.storage.FileScanner; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import java.io.IOException; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java index c15d20b5e1..10b186be69 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java @@ -18,15 +18,15 @@ package org.apache.tajo.storage.avro; -import java.io.IOException; - import org.apache.avro.Schema; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.storage.StorageConstants; + +import java.io.IOException; public class AvroUtil { public static Schema getAvroSchema(TableMeta meta, Configuration conf) diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/package-info.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/package-info.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/avro/package-info.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/package-info.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index 4a83dbf8d7..ac413ca381 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.storage.StorageFragmentProtos.*; import org.apache.tajo.util.TUtil; import java.io.IOException; @@ -32,7 +33,6 @@ import java.util.Arrays; import java.util.List; -import static org.apache.tajo.catalog.proto.CatalogProtos.FileFragmentProto; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; public class FileFragment implements Fragment, Comparable, Cloneable { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexMethod.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexMethod.java similarity index 88% rename from tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexMethod.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexMethod.java index ccba3be6e7..a6af19b5e7 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexMethod.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexMethod.java @@ -20,14 +20,13 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.storage.BaseTupleComparator; import org.apache.tajo.storage.TupleComparator; import java.io.IOException; public interface IndexMethod { IndexWriter getIndexWriter(final Path fileName, int level, Schema keySchema, - TupleComparator comparator) throws IOException; + TupleComparator comparator) throws IOException; IndexReader getIndexReader(final Path fileName, Schema keySchema, - TupleComparator comparator) throws IOException; + TupleComparator comparator) throws IOException; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexReader.java similarity index 97% rename from tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexReader.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexReader.java index 7baf7aa57b..3ae5c9de47 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexReader.java @@ -29,7 +29,7 @@ public interface IndexReader { * * @param key * @return - * @throws IOException + * @throws java.io.IOException */ public long find(Tuple key) throws IOException; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexWriter.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/index/IndexWriter.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/IndexWriter.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/index/OrderIndexReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/OrderIndexReader.java similarity index 95% rename from tajo-storage/src/main/java/org/apache/tajo/storage/index/OrderIndexReader.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/OrderIndexReader.java index 688bbc7c40..0c07b4a809 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/index/OrderIndexReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/OrderIndexReader.java @@ -32,14 +32,14 @@ public interface OrderIndexReader extends IndexReader { * * @param key to find * @return - * @throws IOException + * @throws java.io.IOException */ public long find(Tuple key, boolean nextKey) throws IOException; - + /** * Return the next offset from the latest find or next offset * @return - * @throws IOException + * @throws java.io.IOException */ public long next() throws IOException; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java index f093f9df42..81a1ffd2cc 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java @@ -101,7 +101,7 @@ public class BSTIndexWriter extends IndexWriter implements Closeable { * * @param level * : IndexCreater.ONE_LEVEL_INDEX or IndexCreater.TWO_LEVEL_INDEX - * @throws IOException + * @throws java.io.IOException */ public BSTIndexWriter(final Path fileName, int level, Schema keySchema, TupleComparator comparator) throws IOException { @@ -183,7 +183,7 @@ public void close() throws IOException { int entryNum = keySet.size(); writeHeader(entryNum); - + int loadCount = this.loadNum - 1; for (Tuple key : keySet) { @@ -198,7 +198,7 @@ public void close() throws IOException { byte[] buf = rowStoreEncoder.toBytes(key); out.writeInt(buf.length); out.write(buf); - + /**/ LinkedList offsetList = keyOffsetMap.get(key); /* offset num writing */ @@ -306,7 +306,7 @@ public class BSTIndexReader implements OrderIndexReader , Closeable{ * @param fileName * @param keySchema * @param comparator - * @throws IOException + * @throws java.io.IOException */ public BSTIndexReader(final Path fileName, Schema keySchema, TupleComparator comparator) throws IOException { this.fileName = fileName; @@ -401,7 +401,7 @@ private void fillData() throws IOException { /** * * @return - * @throws IOException + * @throws java.io.IOException */ public long find(Tuple key) throws IOException { return find(key, false); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java index b10d423156..95e8d34bca 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java @@ -18,19 +18,18 @@ package org.apache.tajo.storage.parquet; -import org.apache.tajo.QueryUnitAttemptId; -import org.apache.tajo.storage.StorageConstants; -import parquet.hadoop.ParquetOutputFormat; -import parquet.hadoop.metadata.CompressionCodecName; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.storage.FileAppender; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.TableStatistics; import org.apache.tajo.storage.Tuple; +import parquet.hadoop.ParquetOutputFormat; +import parquet.hadoop.metadata.CompressionCodecName; import java.io.IOException; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoParquetReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoParquetReader.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoParquetReader.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoParquetReader.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoParquetWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoParquetWriter.java similarity index 97% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoParquetWriter.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoParquetWriter.java index 69b76c492d..5f220c5467 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoParquetWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoParquetWriter.java @@ -42,7 +42,7 @@ public class TajoParquetWriter extends ParquetWriter { * @param blockSize The block size threshold. * @param pageSize See parquet write up. Blocks are subdivided into pages * for alignment. - * @throws IOException + * @throws java.io.IOException */ public TajoParquetWriter(Path file, Schema schema, @@ -68,7 +68,7 @@ public TajoParquetWriter(Path file, * for alignment. * @param enableDictionary Whether to use a dictionary to compress columns. * @param validating Whether to turn on validation. - * @throws IOException + * @throws java.io.IOException */ public TajoParquetWriter(Path file, Schema schema, @@ -92,7 +92,7 @@ public TajoParquetWriter(Path file, * * @param file The Path of the file to write to. * @param schema The Tajo schema of the table. - * @throws IOException + * @throws java.io.IOException */ public TajoParquetWriter(Path file, Schema schema) throws IOException { this(file, diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java similarity index 93% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java index 269f782695..a64e9871f9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java @@ -18,22 +18,20 @@ package org.apache.tajo.storage.parquet; -import java.util.Map; - +import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.storage.Tuple; import parquet.Log; import parquet.hadoop.api.InitContext; import parquet.hadoop.api.ReadSupport; import parquet.io.api.RecordMaterializer; import parquet.schema.MessageType; -import org.apache.hadoop.conf.Configuration; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.json.CatalogGsonHelper; -import org.apache.tajo.storage.Tuple; +import java.util.Map; /** - * Tajo implementation of {@link ReadSupport} for {@link Tuple}s. - * Users should use {@link ParquetScanner} and not this class directly. + * Tajo implementation of {@link parquet.hadoop.api.ReadSupport} for {@link org.apache.tajo.storage.Tuple}s. + * Users should use {@link org.apache.tajo.storage.parquet.ParquetScanner} and not this class directly. */ public class TajoReadSupport extends ReadSupport { private static final Log LOG = Log.getLog(TajoReadSupport.class); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java index 7c3d79d778..4375fa4d0b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java @@ -18,29 +18,23 @@ package org.apache.tajo.storage.parquet; -import com.google.protobuf.Message; import com.google.protobuf.InvalidProtocolBufferException; - -import java.nio.ByteBuffer; - -import parquet.io.api.GroupConverter; -import parquet.io.api.Converter; -import parquet.io.api.PrimitiveConverter; -import parquet.io.api.Binary; -import parquet.schema.Type; -import parquet.schema.GroupType; - +import com.google.protobuf.Message; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.*; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.BlobDatum; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.datum.ProtobufDatumFactory; +import parquet.io.api.Binary; +import parquet.io.api.Converter; +import parquet.io.api.GroupConverter; +import parquet.io.api.PrimitiveConverter; +import parquet.schema.GroupType; +import parquet.schema.Type; + +import java.nio.ByteBuffer; /** * Converter to convert a Parquet record into a Tajo Tuple. diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoRecordMaterializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordMaterializer.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoRecordMaterializer.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordMaterializer.java index e31828c318..436159ce81 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoRecordMaterializer.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordMaterializer.java @@ -18,14 +18,13 @@ package org.apache.tajo.storage.parquet; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.storage.Tuple; import parquet.io.api.GroupConverter; import parquet.io.api.RecordMaterializer; import parquet.schema.MessageType; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.storage.Tuple; - /** * Materializes a Tajo Tuple from a stream of Parquet data. */ diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java similarity index 89% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java index 259223189b..555b623f1c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java @@ -157,48 +157,48 @@ private Type convertColumn(Column column) { switch (type) { case BOOLEAN: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.BOOLEAN); + PrimitiveTypeName.BOOLEAN); case BIT: case INT2: case INT4: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.INT32); + PrimitiveTypeName.INT32); case INT8: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.INT64); + PrimitiveTypeName.INT64); case FLOAT4: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.FLOAT); + PrimitiveTypeName.FLOAT); case FLOAT8: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.DOUBLE); + PrimitiveTypeName.DOUBLE); case CHAR: case TEXT: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.BINARY, + PrimitiveTypeName.BINARY, OriginalType.UTF8); case PROTOBUF: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.BINARY); + PrimitiveTypeName.BINARY); case BLOB: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.BINARY); + PrimitiveTypeName.BINARY); case INET4: case INET6: return primitive(column.getSimpleName(), - PrimitiveType.PrimitiveTypeName.BINARY); + PrimitiveTypeName.BINARY); default: throw new RuntimeException("Cannot convert Tajo type: " + type); } } private PrimitiveType primitive(String name, - PrimitiveType.PrimitiveTypeName primitive) { + PrimitiveTypeName primitive) { return new PrimitiveType(Type.Repetition.OPTIONAL, primitive, name, null); } private PrimitiveType primitive(String name, - PrimitiveType.PrimitiveTypeName primitive, + PrimitiveTypeName primitive, OriginalType originalType) { return new PrimitiveType(Type.Repetition.OPTIONAL, primitive, name, originalType); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java similarity index 95% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java index 35165de791..00aadf4cac 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java @@ -18,10 +18,12 @@ package org.apache.tajo.storage.parquet; -import java.util.Map; -import java.util.HashMap; -import java.util.List; - +import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.storage.Tuple; import parquet.hadoop.api.WriteSupport; import parquet.io.api.Binary; import parquet.io.api.RecordConsumer; @@ -29,15 +31,12 @@ import parquet.schema.MessageType; import parquet.schema.Type; -import org.apache.hadoop.conf.Configuration; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.datum.Datum; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** - * Tajo implementation of {@link WriteSupport} for {@link Tuple}s. + * Tajo implementation of {@link parquet.hadoop.api.WriteSupport} for {@link org.apache.tajo.storage.Tuple}s. * Users should use {@link ParquetAppender} and not this class directly. */ public class TajoWriteSupport extends WriteSupport { @@ -70,7 +69,7 @@ public WriteContext init(Configuration configuration) { /** * Called once per row group. * - * @param recordConsumer The {@link RecordConsumer} to write to. + * @param recordConsumer The {@link parquet.io.api.RecordConsumer} to write to. */ @Override public void prepareForWrite(RecordConsumer recordConsumer) { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/package-info.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/package-info.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/parquet/package-info.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/package-info.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/BytesRefArrayWritable.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/BytesRefArrayWritable.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/BytesRefArrayWritable.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/BytesRefArrayWritable.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/BytesRefWritable.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/BytesRefWritable.java similarity index 97% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/BytesRefWritable.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/BytesRefWritable.java index c83b50562f..158c740784 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/BytesRefWritable.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/BytesRefWritable.java @@ -102,7 +102,7 @@ private void lazyDecompress() throws IOException { * Returns a copy of the underlying bytes referenced by this instance. * * @return a new copied byte array - * @throws IOException + * @throws java.io.IOException */ public byte[] getBytesCopy() throws IOException { lazyDecompress(); @@ -114,7 +114,7 @@ public byte[] getBytesCopy() throws IOException { /** * Returns the underlying bytes. * - * @throws IOException + * @throws java.io.IOException */ public byte[] getData() throws IOException { lazyDecompress(); @@ -125,7 +125,7 @@ public byte[] getData() throws IOException { * readFields() will corrupt the array. So use the set method whenever * possible. * - * @see #readFields(DataInput) + * @see #readFields(java.io.DataInput) */ public void set(byte[] newData, int offset, int len) { bytes = newData; @@ -138,7 +138,7 @@ public void set(byte[] newData, int offset, int len) { * readFields() will corrupt the array. So use the set method whenever * possible. * - * @see #readFields(DataInput) + * @see #readFields(java.io.DataInput) */ public void set(LazyDecompressionCallback newData, int offset, int len) { bytes = null; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/LazyDecompressionCallback.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/LazyDecompressionCallback.java similarity index 94% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/LazyDecompressionCallback.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/LazyDecompressionCallback.java index 707d55aa56..eab23566f9 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/LazyDecompressionCallback.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/LazyDecompressionCallback.java @@ -23,7 +23,7 @@ /** * Used to call back lazy decompression process. * - * @see BytesRefWritable + * @see org.apache.tajo.storage.rcfile.BytesRefWritable */ public interface LazyDecompressionCallback { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayInputStream.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayInputStream.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayInputStream.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayInputStream.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayOutputStream.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayOutputStream.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayOutputStream.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncByteArrayOutputStream.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataInputBuffer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataInputBuffer.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataInputBuffer.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataInputBuffer.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataOutputBuffer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataOutputBuffer.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataOutputBuffer.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/NonSyncDataOutputBuffer.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java index 23815d9f2d..d88223be33 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java @@ -39,7 +39,6 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import java.io.Closeable; @@ -63,7 +62,7 @@ * which can be set like: conf.setInt(COLUMNS_BUFFER_SIZE_CONF_STR, * 4 * 1024 * 1024) . *

- * RCFile provides {@link Writer}, {@link Reader} and classes for + * RCFile provides {@link java.io.Writer}, {@link java.io.Reader} and classes for * writing, reading respectively. *

*

@@ -79,11 +78,11 @@ * RCFile compresses values in a more fine-grained manner then record level * compression. However, It currently does not support compress the key part * yet. The actual compression algorithm used to compress key and/or values can - * be specified by using the appropriate {@link CompressionCodec}. + * be specified by using the appropriate {@link org.apache.hadoop.io.compress.CompressionCodec}. *

*

*

- * The {@link Reader} is used to read and explain the bytes of RCFile. + * The {@link java.io.Reader} is used to read and explain the bytes of RCFile. *

*

*

RCFile Formats

@@ -97,7 +96,7 @@ * keys/values in this file. *
  • compression codec - CompressionCodec class which is used * for compression of keys and/or values (if compression is enabled).
    • - *
  • metadata - {@link Metadata} for this file.
    • + *
  • metadata - {@link org.apache.hadoop.io.SequenceFile.Metadata} for this file.
    • *
  • sync - A sync marker to denote end of the header.
    • * *

    @@ -881,13 +880,13 @@ public void addTuple(Tuple t) throws IOException { /** * Append a row of values. Currently it only can accept < - * {@link Tuple}. If its size() is less than the + * {@link org.apache.tajo.storage.Tuple}. If its size() is less than the * column number in the file, zero bytes are appended for the empty columns. * If its size() is greater then the column number in the file, the exceeded * columns' bytes are ignored. * * @param tuple a Tuple with the list of serialized columns - * @throws IOException + * @throws java.io.IOException */ public void append(Tuple tuple) throws IOException { int size = schema.size(); @@ -923,7 +922,7 @@ public void append(Tuple tuple) throws IOException { * get number of bytes to store the keyBuffer. * * @return number of bytes used to store this KeyBuffer on disk - * @throws IOException + * @throws java.io.IOException */ public int getKeyBufferSize() throws IOException { int ret = 0; @@ -943,7 +942,7 @@ public int getKeyBufferSize() throws IOException { * get number of bytes to store the key part. * * @return number of bytes used to store this Key part on disk - * @throws IOException + * @throws java.io.IOException */ public int getKeyPartSize() throws IOException { int ret = 12; //12 bytes |record count, key length, compressed key length| @@ -1296,7 +1295,7 @@ public Text getMetadataValueOf(Text key) { /** * Override this method to specialize the type of - * {@link FSDataInputStream} returned. + * {@link org.apache.hadoop.fs.FSDataInputStream} returned. */ protected FSDataInputStream openFile(FileSystem fs, Path file, int bufferSize) throws IOException { return fs.open(file, bufferSize); @@ -1400,10 +1399,10 @@ public long getPosition() throws IOException { *

    *

    * The position passed must be a position returned by - * {@link RCFile.RCFileAppender#getLength()} when writing this file. To seek to an - * arbitrary position, use {@link RCFile.RCFileScanner#sync(long)}. In another + * {@link org.apache.tajo.storage.rcfile.RCFile.RCFileAppender#getLength()} when writing this file. To seek to an + * arbitrary position, use {@link org.apache.tajo.storage.rcfile.RCFile.RCFileScanner#sync(long)}. In another * words, the current seek can only seek to the end of the file. For other - * positions, use {@link RCFile.RCFileScanner#sync(long)}. + * positions, use {@link org.apache.tajo.storage.rcfile.RCFile.RCFileScanner#sync(long)}. */ public void seek(long position) throws IOException { in.seek(position); @@ -1491,7 +1490,7 @@ private KeyBuffer createKeyBuffer() { * block. * * @return the length of the next record or -1 if there is no next record - * @throws IOException + * @throws java.io.IOException */ private int readRecordLength() throws IOException { if (in.getPos() >= end) { @@ -1669,7 +1668,7 @@ public float getProgress() { * value buffer. * * @return next row number - * @throws IOException + * @throws java.io.IOException */ public boolean nextBuffer(LongWritable readRows) throws IOException { if (readRowsIndexInBuffer < recordsNumInValBuffer) { @@ -1695,7 +1694,7 @@ public boolean nextBuffer(LongWritable readRows) throws IOException { * get the current row used,make sure called {@link #next()} * first. * - * @throws IOException + * @throws java.io.IOException */ public void getCurrentRow(Tuple tuple) throws IOException { if (!keyInit || rowFetched) { @@ -1730,7 +1729,7 @@ public void getCurrentRow(Tuple tuple) throws IOException { * @param selCol - index among selectedColumns * @param col - column object to update the state of. prvLength will be * set to the new read position - * @throws IOException + * @throws java.io.IOException */ private void colAdvanceRow(int selCol, SelectedColumn col) throws IOException { if (col.runLength > 0) { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionInputStream.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionInputStream.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionInputStream.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionInputStream.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionOutputStream.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionOutputStream.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionOutputStream.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/SchemaAwareCompressionOutputStream.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java similarity index 95% rename from tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java index 543336f0eb..f76593ed0f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java @@ -18,6 +18,13 @@ package org.apache.tajo.storage.thirdparty.parquet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.*; +import org.apache.hadoop.util.ReflectionUtils; +import parquet.bytes.BytesInput; +import parquet.hadoop.BadConfigurationException; +import parquet.hadoop.metadata.CompressionCodecName; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,19 +32,6 @@ import java.util.HashMap; import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.compress.CodecPool; -import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.io.compress.CompressionOutputStream; -import org.apache.hadoop.io.compress.Compressor; -import org.apache.hadoop.io.compress.Decompressor; -import org.apache.hadoop.util.ReflectionUtils; - -import parquet.bytes.BytesInput; -import parquet.hadoop.BadConfigurationException; -import parquet.hadoop.metadata.CompressionCodecName; - class CodecFactory { public class BytesDecompressor { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java index 5f89ead8f3..0dedd9bac6 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java @@ -18,17 +18,6 @@ package org.apache.tajo.storage.thirdparty.parquet; -import static org.apache.tajo.storage.thirdparty.parquet.CodecFactory.BytesCompressor; -import static parquet.Log.INFO; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - import parquet.Log; import parquet.bytes.BytesInput; import parquet.bytes.CapacityByteArrayOutputStream; @@ -37,12 +26,18 @@ import parquet.column.page.DictionaryPage; import parquet.column.page.PageWriteStore; import parquet.column.page.PageWriter; -import parquet.column.statistics.Statistics; import parquet.column.statistics.BooleanStatistics; +import parquet.column.statistics.Statistics; import parquet.format.converter.ParquetMetadataConverter; import parquet.io.ParquetEncodingException; import parquet.schema.MessageType; +import java.io.IOException; +import java.util.*; + +import static org.apache.tajo.storage.thirdparty.parquet.CodecFactory.BytesCompressor; +import static parquet.Log.INFO; + class ColumnChunkPageWriteStore implements PageWriteStore { private static final Log LOG = Log.getLog(ColumnChunkPageWriteStore.class); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordReader.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordReader.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordReader.java index 61567e5714..6bbd7b54b2 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordReader.java @@ -18,9 +18,6 @@ package org.apache.tajo.storage.thirdparty.parquet; -import java.io.IOException; -import java.util.List; -import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import parquet.Log; @@ -39,6 +36,10 @@ import parquet.schema.MessageType; import parquet.schema.Type; +import java.io.IOException; +import java.util.List; +import java.util.Map; + import static java.lang.String.format; import static parquet.Log.DEBUG; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java index 7410d2bddb..532d9a2427 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java @@ -18,16 +18,6 @@ package org.apache.tajo.storage.thirdparty.parquet; -import static java.lang.Math.max; -import static java.lang.Math.min; -import static java.lang.String.format; -import static org.apache.tajo.storage.thirdparty.parquet.CodecFactory.BytesCompressor; -import static parquet.Log.DEBUG; -import static parquet.Preconditions.checkNotNull; - -import java.io.IOException; -import java.util.Map; - import parquet.Log; import parquet.column.ParquetProperties.WriterVersion; import parquet.column.impl.ColumnWriteStoreImpl; @@ -36,6 +26,16 @@ import parquet.io.MessageColumnIO; import parquet.schema.MessageType; +import java.io.IOException; +import java.util.Map; + +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.lang.String.format; +import static org.apache.tajo.storage.thirdparty.parquet.CodecFactory.BytesCompressor; +import static parquet.Log.DEBUG; +import static parquet.Preconditions.checkNotNull; + class InternalParquetRecordWriter { private static final Log LOG = Log.getLog(InternalParquetRecordWriter.class); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java similarity index 95% rename from tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java index 73ce7c2ab6..f1c53682f0 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java @@ -18,24 +18,10 @@ package org.apache.tajo.storage.thirdparty.parquet; -import static parquet.Log.DEBUG; -import static parquet.format.Util.writeFileMetaData; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; - import parquet.Log; import parquet.Version; import parquet.bytes.BytesInput; @@ -45,17 +31,19 @@ import parquet.column.statistics.Statistics; import parquet.format.converter.ParquetMetadataConverter; import parquet.hadoop.Footer; -import parquet.hadoop.metadata.BlockMetaData; -import parquet.hadoop.metadata.ColumnChunkMetaData; -import parquet.hadoop.metadata.ColumnPath; -import parquet.hadoop.metadata.CompressionCodecName; -import parquet.hadoop.metadata.FileMetaData; -import parquet.hadoop.metadata.GlobalMetaData; -import parquet.hadoop.metadata.ParquetMetadata; +import parquet.hadoop.metadata.*; import parquet.io.ParquetEncodingException; import parquet.schema.MessageType; import parquet.schema.PrimitiveType.PrimitiveTypeName; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.*; +import java.util.Map.Entry; + +import static parquet.Log.DEBUG; +import static parquet.format.Util.writeFileMetaData; + /** * Internal implementation of the Parquet file writer as a block container * @@ -148,7 +136,7 @@ private final STATE error() throws IOException { * @param schema the schema of the data * @param out the file to write to * @param codec the codec to use to compress blocks - * @throws IOException if the file can not be created + * @throws java.io.IOException if the file can not be created */ public ParquetFileWriter(Configuration configuration, MessageType schema, Path file) throws IOException { super(); @@ -159,7 +147,7 @@ public ParquetFileWriter(Configuration configuration, MessageType schema, Path f /** * start the file - * @throws IOException + * @throws java.io.IOException */ public void start() throws IOException { state = state.start(); @@ -170,7 +158,7 @@ public void start() throws IOException { /** * start a block * @param recordCount the record count in this block - * @throws IOException + * @throws java.io.IOException */ public void startBlock(long recordCount) throws IOException { state = state.startBlock(); @@ -186,7 +174,7 @@ public void startBlock(long recordCount) throws IOException { * @param valueCount the value count in this column * @param statistics the statistics in this column * @param compressionCodecName - * @throws IOException + * @throws java.io.IOException */ public void startColumn(ColumnDescriptor descriptor, long valueCount, @@ -312,7 +300,7 @@ public void writeDataPage( * @param bytes bytes to be written including page headers * @param uncompressedTotalPageSize total uncompressed size (without page headers) * @param compressedTotalPageSize total compressed size (without page headers) - * @throws IOException + * @throws java.io.IOException */ void writeDataPages(BytesInput bytes, long uncompressedTotalPageSize, @@ -332,7 +320,7 @@ void writeDataPages(BytesInput bytes, /** * end a column (once all rep, def and data have been written) - * @throws IOException + * @throws java.io.IOException */ public void endColumn() throws IOException { state = state.endColumn(); @@ -357,7 +345,7 @@ public void endColumn() throws IOException { /** * ends a block once all column chunks have been written - * @throws IOException + * @throws java.io.IOException */ public void endBlock() throws IOException { state = state.endBlock(); @@ -371,7 +359,7 @@ public void endBlock() throws IOException { * ends a file once all blocks have been written. * closes the file. * @param extraMetaData the extra meta data to write in the footer - * @throws IOException + * @throws java.io.IOException */ public void end(Map extraMetaData) throws IOException { state = state.end(); @@ -395,7 +383,7 @@ private static void serializeFooter(ParquetMetadata footer, FSDataOutputStream o * @param configuration the configuration to use to get the FileSystem * @param outputPath the directory to write the _metadata file to * @param footers the list of footers to merge - * @throws IOException + * @throws java.io.IOException */ public static void writeMetadataFile(Configuration configuration, Path outputPath, List