From ee530a00cec9fafcab26267d409197c5484f52dd Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 18 May 2015 13:37:38 -0700 Subject: [PATCH 1/2] TAJO-1613: Rename StorageManager to Tablespace. --- .../engine/planner/PhysicalPlannerImpl.java | 6 +- .../physical/ColPartitionStoreExec.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 4 +- .../physical/RangeShuffleFileWriteExec.java | 2 +- .../engine/planner/physical/SeqScanExec.java | 4 +- .../planner/physical/StoreTableExec.java | 2 +- .../org/apache/tajo/master/GlobalEngine.java | 4 +- .../org/apache/tajo/master/TajoMaster.java | 6 +- .../apache/tajo/master/exec/DDLExecutor.java | 6 +- .../NonForwardQueryResultFileScanner.java | 16 ++-- .../tajo/master/exec/QueryExecutor.java | 4 +- .../tajo/querymaster/QueryMasterTask.java | 4 +- .../tajo/querymaster/Repartitioner.java | 32 ++++---- .../org/apache/tajo/querymaster/Stage.java | 12 +-- .../apache/tajo/worker/TajoQueryEngine.java | 1 - .../java/org/apache/tajo/worker/Task.java | 2 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../planner/global/TestBroadcastJoinPlan.java | 2 +- .../planner/physical/TestBNLJoinExec.java | 12 +-- .../planner/physical/TestBSTIndexExec.java | 6 +- .../physical/TestExternalSortExec.java | 4 +- .../physical/TestFullOuterHashJoinExec.java | 24 +++--- .../physical/TestFullOuterMergeJoinExec.java | 38 ++++----- .../physical/TestHashAntiJoinExec.java | 8 +- .../planner/physical/TestHashJoinExec.java | 12 +-- .../physical/TestHashSemiJoinExec.java | 8 +- .../physical/TestLeftOuterHashJoinExec.java | 28 +++---- .../planner/physical/TestMergeJoinExec.java | 8 +- .../planner/physical/TestNLJoinExec.java | 12 +-- .../planner/physical/TestPhysicalPlanner.java | 56 ++++++------- .../TestProgressExternalSortExec.java | 4 +- .../physical/TestRightOuterHashJoinExec.java | 18 ++--- .../physical/TestRightOuterMergeJoinExec.java | 36 ++++----- .../engine/planner/physical/TestSortExec.java | 8 +- .../tajo/engine/query/TestHBaseTable.java | 22 +++--- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../org/apache/tajo/jdbc/TestResultSet.java | 4 +- .../tajo/master/TestExecutionBlockCursor.java | 4 +- .../org/apache/tajo/storage/TestRowFile.java | 4 +- .../org/apache/tajo/storage/MergeScanner.java | 2 +- .../org/apache/tajo/storage/TableSpace.java | 74 ------------------ .../tajo/storage/TableSpaceManager.java | 30 +++---- .../{StorageManager.java => Tablespace.java} | 35 +++------ .../src/main/resources/storage-default.xml | 4 +- .../src/test/resources/storage-default.xml | 4 +- .../tajo/storage/hbase/HBasePutAppender.java | 4 +- .../tajo/storage/hbase/HBaseScanner.java | 4 +- ...orageManager.java => HBaseTablespace.java} | 13 ++-- .../hbase/TestHBaseStorageManager.java | 4 +- .../org/apache/tajo/storage/FileAppender.java | 2 +- ...torageManager.java => FileTablespace.java} | 78 ++++--------------- .../storage/HashShuffleAppenderManager.java | 2 +- .../tajo/storage/TestCompressionStorages.java | 2 +- .../tajo/storage/TestFileStorageManager.java | 10 +-- .../apache/tajo/storage/TestFileSystems.java | 4 +- .../apache/tajo/storage/TestMergeScanner.java | 2 +- .../org/apache/tajo/storage/TestStorages.java | 28 +++---- .../tajo/storage/index/TestBSTIndex.java | 20 ++--- .../index/TestSingleCSVFileBSTIndex.java | 4 +- .../src/test/resources/storage-default.xml | 4 +- 60 files changed, 313 insertions(+), 446 deletions(-) delete mode 100644 tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpace.java rename tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/{StorageManager.java => Tablespace.java} (94%) rename tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/{HBaseStorageManager.java => HBaseTablespace.java} (98%) rename tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/{FileStorageManager.java => FileTablespace.java} (95%) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index ff42d5fe90..0252051613 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -251,7 +251,7 @@ public long estimateSizeRecursive(TaskAttemptContext ctx, String [] tableIds) th FragmentProto[] fragmentProtos = ctx.getTables(tableId); List fragments = FragmentConvertor.convert(ctx.getConf(), fragmentProtos); for (Fragment frag : fragments) { - size += StorageManager.getFragmentLength(ctx.getConf(), frag); + size += Tablespace.getFragmentLength(ctx.getConf(), frag); } } return size; @@ -924,7 +924,7 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St if (broadcastFlag) { PartitionedTableScanNode partitionedTableScanNode = (PartitionedTableScanNode) scanNode; List fileFragments = TUtil.newList(); - FileStorageManager fileStorageManager = (FileStorageManager) TableSpaceManager.getFileStorageManager(ctx.getConf()); + FileTablespace fileStorageManager = (FileTablespace) TableSpaceManager.getFileStorageManager(ctx.getConf()); for (Path path : partitionedTableScanNode.getInputPaths()) { fileFragments.addAll(TUtil.newList(fileStorageManager.split(scanNode.getCanonicalName(), path))); } @@ -1188,7 +1188,7 @@ public PhysicalExec createIndexScanExec(TaskAttemptContext ctx, FragmentConvertor.convert(ctx.getConf(), fragmentProtos); String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys()); - FileStorageManager sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(ctx.getConf()); + FileTablespace sm = (FileTablespace) TableSpaceManager.getFileStorageManager(ctx.getConf()); Path indexPath = new Path(sm.getTablePath(annotation.getTableName()), "index"); TupleComparator comp = new BaseTupleComparator(annotation.getKeySchema(), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 8d53a6f934..3121671b0c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -165,7 +165,7 @@ public void openAppender(int suffixId) throws IOException { actualFilePath = new Path(lastFileName + "_" + suffixId); } - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(context.getConf())) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(context.getConf())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 92bf3b5eed..d240edbd2e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -35,7 +35,7 @@ import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.FileStorageManager; +import org.apache.tajo.storage.FileTablespace; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; @@ -141,7 +141,7 @@ private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List 0) { for (FileStatus eachFile : files) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index 6fd2ce4b59..7bf9e66f96 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -77,7 +77,7 @@ public void init() throws IOException { context.getDataChannel().getStoreType() : "RAW"); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); - this.appender = (FileAppender) ((FileStorageManager) TableSpaceManager.getFileStorageManager(context.getConf())) + this.appender = (FileAppender) ((FileTablespace) TableSpaceManager.getFileStorageManager(context.getConf())) .getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(); this.appender.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index 2225daef90..c98c74127c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -202,9 +202,9 @@ private void initScanner(Schema projected) throws IOException { FragmentConvertor.convert(context.getConf(), fragments), projected ); } else { - StorageManager storageManager = TableSpaceManager.getStorageManager( + Tablespace tablespace = TableSpaceManager.getStorageManager( context.getConf(), plan.getTableDesc().getMeta().getStoreType()); - this.scanner = storageManager.getScanner(meta, + this.scanner = tablespace.getScanner(meta, plan.getPhysicalSchema(), fragments[0], projected); } scanner.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index b0263d7ebe..5b17eee70d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -90,7 +90,7 @@ public void openNewFile(int suffixId) throws IOException { lastFileName = new Path(lastFileName + "_" + suffixId); } - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(context.getConf())) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(context.getConf())) .getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index aeb41663e2..2cd585ffde 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -53,7 +53,7 @@ import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; import org.apache.tajo.plan.verifier.VerificationState; import org.apache.tajo.plan.verifier.VerifyException; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.CommonTestingUtil; @@ -68,7 +68,7 @@ public class GlobalEngine extends AbstractService { private final static Log LOG = LogFactory.getLog(GlobalEngine.class); private final MasterContext context; - private final StorageManager sm; + private final Tablespace sm; private SQLAnalyzer analyzer; private CatalogService catalog; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index abf070e945..c41fdde0d9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -53,7 +53,7 @@ import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.session.SessionManager; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.*; import org.apache.tajo.util.history.HistoryReader; @@ -114,7 +114,7 @@ public class TajoMaster extends CompositeService { private CatalogServer catalogServer; private CatalogService catalog; - private StorageManager storeManager; + private Tablespace storeManager; private GlobalEngine globalEngine; private AsyncDispatcher dispatcher; private TajoMasterClientService tajoMasterClientService; @@ -477,7 +477,7 @@ public GlobalEngine getGlobalEngine() { return globalEngine; } - public StorageManager getStorageManager() { + public Tablespace getStorageManager() { return storeManager; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 888194d19c..93c950ebfe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -36,7 +36,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TableSpaceManager; @@ -54,12 +54,12 @@ public class DDLExecutor { private final TajoMaster.MasterContext context; private final CatalogService catalog; - private final StorageManager storageManager; + private final Tablespace tablespace; public DDLExecutor(TajoMaster.MasterContext context) { this.context = context; this.catalog = context.getCatalog(); - this.storageManager = context.getStorageManager(); + this.tablespace = context.getStorageManager(); } public boolean execute(QueryContext queryContext, LogicalPlan plan) throws IOException { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index d2022b55c9..8f6c6f98a8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -78,11 +78,11 @@ public void init() throws IOException { /** * Set partition path and depth if ScanNode's qualification exists * - * @param storageManager target storage manager to be set with partition info + * @param tablespace target storage manager to be set with partition info */ - private void setPartition(StorageManager storageManager) { + private void setPartition(Tablespace tablespace) { if (tableDesc.isExternal() && tableDesc.hasPartition() && scanNode.getQual() != null && - storageManager instanceof FileStorageManager) { + tablespace instanceof FileTablespace) { StringBuffer path = new StringBuffer(); int depth = 0; if (tableDesc.hasPartition()) { @@ -94,17 +94,17 @@ private void setPartition(StorageManager storageManager) { depth++; } } - ((FileStorageManager)storageManager).setPartitionPath(path.toString()); - ((FileStorageManager)storageManager).setCurrentDepth(depth); + ((FileTablespace) tablespace).setPartitionPath(path.toString()); + ((FileTablespace) tablespace).setCurrentDepth(depth); scanNode.setQual(null); } } private void initSeqScanExec() throws IOException { - StorageManager storageManager = TableSpaceManager.getStorageManager(tajoConf, tableDesc.getMeta().getStoreType()); + Tablespace tablespace = TableSpaceManager.getStorageManager(tajoConf, tableDesc.getMeta().getStoreType()); List fragments = null; - setPartition(storageManager); - fragments = storageManager.getNonForwardSplit(tableDesc, currentFragmentIndex, MAX_FRAGMENT_NUM_PER_SCAN); + setPartition(tablespace); + fragments = tablespace.getNonForwardSplit(tableDesc, currentFragmentIndex, MAX_FRAGMENT_NUM_PER_SCAN); if (fragments != null && !fragments.isEmpty()) { FragmentProto[] fragmentProtos = FragmentConvertor.toFragmentProtoArray(fragments.toArray(new Fragment[] {})); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 695c38e4f0..6ab096f543 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -438,7 +438,7 @@ public void executeDistributedQuery(QueryContext queryContext, Session session, String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { - StorageManager sm = TableSpaceManager.getStorageManager(context.getConf(), storeType); + Tablespace sm = TableSpaceManager.getStorageManager(context.getConf(), storeType); StorageProperty storageProperty = sm.getStorageProperty(); if (!storageProperty.isSupportsInsertInto()) { throw new VerifyException("Inserting into non-file storage is not supported."); @@ -476,7 +476,7 @@ public static MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext contex String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { - StorageManager sm = TableSpaceManager.getStorageManager(planner.getConf(), storeType); + Tablespace sm = TableSpaceManager.getStorageManager(planner.getConf(), storeType); StorageProperty storageProperty = sm.getStorageProperty(); if (storageProperty.isSortedInsert()) { String tableName = PlannerUtil.getStoreTableName(plan); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index d77001c8c5..a48f0a009f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -55,7 +55,7 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.session.Session; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.StorageProperty; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TableSpaceManager; @@ -308,7 +308,7 @@ private static boolean isTerminatedState(QueryState state) { } public synchronized void startQuery() { - StorageManager sm = null; + Tablespace sm = null; LogicalPlan plan = null; try { if (query != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index b43b506973..0e1b501d10 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -91,8 +91,8 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC for (int i = 0; i < scans.length; i++) { TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName()); if (tableDesc == null) { // if it is a real table stored on storage - FileStorageManager storageManager = - (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); + FileTablespace storageManager = + (FileTablespace) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); tablePath = storageManager.getTablePath(scans[i].getTableName()); if (execBlock.getUnionScanMap() != null && !execBlock.getUnionScanMap().isEmpty()) { @@ -112,13 +112,13 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC throw new IOException(e); } - StorageManager storageManager = + Tablespace tablespace = TableSpaceManager.getStorageManager(stage.getContext().getConf(), tableDesc.getMeta().getStoreType()); - // if table has no data, storageManager will return empty FileFragment. + // if table has no data, tablespace will return empty FileFragment. // So, we need to handle FileFragment by its size. // If we don't check its size, it can cause IndexOutOfBoundsException. - List fileFragments = storageManager.getSplits(scans[i].getCanonicalName(), tableDesc); + List fileFragments = tablespace.getSplits(scans[i].getCanonicalName(), tableDesc); if (fileFragments.size() > 0) { fragments[i] = fileFragments.get(0); } else { @@ -407,8 +407,8 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster Path[] partitionScanPaths = null; TableDesc tableDesc = masterContext.getTableDescMap().get(eachScan.getCanonicalName()); if (eachScan.getType() == NodeType.PARTITIONS_SCAN) { - FileStorageManager storageManager = - (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); + FileTablespace storageManager = + (FileTablespace) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)eachScan; partitionScanPaths = partitionScan.getInputPaths(); @@ -416,9 +416,9 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster getFragmentsFromPartitionedTable(storageManager, eachScan, tableDesc); partitionScan.setInputPaths(partitionScanPaths); } else { - StorageManager storageManager = TableSpaceManager.getStorageManager(stage.getContext().getConf(), + Tablespace tablespace = TableSpaceManager.getStorageManager(stage.getContext().getConf(), tableDesc.getMeta().getStoreType()); - Collection scanFragments = storageManager.getSplits(eachScan.getCanonicalName(), + Collection scanFragments = tablespace.getSplits(eachScan.getCanonicalName(), tableDesc, eachScan); if (scanFragments != null) { rightFragments.addAll(scanFragments); @@ -488,7 +488,7 @@ public static Map>> merge /** * It creates a number of fragments for all partitions. */ - public static List getFragmentsFromPartitionedTable(FileStorageManager sm, + public static List getFragmentsFromPartitionedTable(FileTablespace sm, ScanNode scan, TableDesc table) throws IOException { if (!(scan instanceof PartitionedTableScanNode)) { @@ -535,14 +535,14 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)scan; partitionScanPaths = partitionScan.getInputPaths(); // set null to inputPaths in getFragmentsFromPartitionedTable() - FileStorageManager storageManager = - (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); + FileTablespace storageManager = + (FileTablespace) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); scanFragments = getFragmentsFromPartitionedTable(storageManager, scan, desc); } else { - StorageManager storageManager = + Tablespace tablespace = TableSpaceManager.getStorageManager(stage.getContext().getConf(), desc.getMeta().getStoreType()); - scanFragments = storageManager.getSplits(scan.getCanonicalName(), desc, scan); + scanFragments = tablespace.getSplits(scan.getCanonicalName(), desc, scan); } if (scanFragments != null) { @@ -645,7 +645,7 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo ExecutionBlock execBlock = stage.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = ((FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf())) + tablePath = ((FileTablespace) TableSpaceManager.getFileStorageManager(stage.getContext().getConf())) .getTablePath(scan.getTableName()); ExecutionBlock sampleChildBlock = masterPlan.getChild(stage.getId(), 0); @@ -811,7 +811,7 @@ public static void scheduleHashShuffledFetches(TaskSchedulerContext schedulerCon ExecutionBlock execBlock = stage.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = ((FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf())) + tablePath = ((FileTablespace) TableSpaceManager.getFileStorageManager(stage.getContext().getConf())) .getTablePath(scan.getTableName()); Fragment frag = new FileFragment(scan.getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 548ea26932..160bfaa815 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -59,8 +59,8 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.querymaster.Task.IntermediateEntry; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.FileTablespace; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; @@ -1089,13 +1089,13 @@ private static void scheduleFragmentsForLeafQuery(Stage stage) throws IOExceptio // span a number of blocks or possibly consists of a number of files. if (scan.getType() == NodeType.PARTITIONS_SCAN) { // After calling this method, partition paths are removed from the physical plan. - FileStorageManager storageManager = - (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); + FileTablespace storageManager = + (FileTablespace) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); fragments = Repartitioner.getFragmentsFromPartitionedTable(storageManager, scan, table); } else { - StorageManager storageManager = + Tablespace tablespace = TableSpaceManager.getStorageManager(stage.getContext().getConf(), meta.getStoreType()); - fragments = storageManager.getSplits(scan.getCanonicalName(), table, scan); + fragments = tablespace.getSplits(scan.getCanonicalName(), table, scan); } Stage.scheduleFragments(stage, fragments); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java index 925c047f5c..b47bf2f654 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java @@ -24,7 +24,6 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.exception.InternalException; -import org.apache.tajo.storage.StorageManager; import java.io.IOException; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 01f56b82d0..4716dccad7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -161,7 +161,7 @@ public void initPlan() throws IOException { this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys()); } } else { - Path outFilePath = ((FileStorageManager) TableSpaceManager.getFileStorageManager(systemConf)) + Path outFilePath = ((FileTablespace) TableSpaceManager.getFileStorageManager(systemConf)) .getAppenderFilePath(taskId, queryContext.getStagingDir()); LOG.info("Output File Path: " + outFilePath); context.setOutputPath(outFilePath); diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index a1c6011ba8..b5be9d0422 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -46,7 +46,7 @@ public class BackendTestingUtil { public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { - FileStorageManager sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); + FileTablespace sm = (FileTablespace) TableSpaceManager.getFileStorageManager(conf); Appender appender; Path filePath = new Path(tablePath, "table.csv"); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 44a22ae324..f0e19b464b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -136,7 +136,7 @@ private TableDesc makeTestData(String tableName, Schema schema, int dataSize) th contentsData += j; } } - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(tableMeta, schema, dataPath); appender.init(); Tuple tuple = new VTuple(schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 104e09b65a..ae282339d0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -86,7 +86,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); @@ -149,10 +149,10 @@ public final void testBNLCrossJoin() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -182,9 +182,9 @@ public final void testBNLInnerJoin() throws IOException, PlanningException { LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context).getRootBlock().getRoot(); - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index a8597e9aa4..62144d2e63 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -65,7 +65,7 @@ public class TestBSTIndexExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private FileStorageManager sm; + private FileTablespace sm; private Schema idxSchema; private BaseTupleComparator comp; private BSTIndex.BSTIndexWriter writer; @@ -90,7 +90,7 @@ public void setup() throws Exception { Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); + sm = (FileTablespace) TableSpaceManager.getFileStorageManager(conf); idxPath = new Path(workDir, "test.idx"); @@ -162,7 +162,7 @@ public void testEqual() throws Exception { this.rndKey = rnd.nextInt(250); final String QUERY = "select * from employee where managerId = " + rndKey; - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testEqual"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 221a622fb9..8aea2b6b57 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -82,7 +82,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); @@ -119,7 +119,7 @@ public void tearDown() throws Exception { @Test public final void testNext() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 501557ffbc..75ae4391f3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -104,7 +104,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender1 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -134,7 +134,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender2 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -174,7 +174,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender3 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -227,7 +227,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender5 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); @@ -266,9 +266,9 @@ public final void testFullOuterHashJoinExec0() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), + FileFragment[] dep3Frags = FileTablespace.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); @@ -305,9 +305,9 @@ public final void testFullOuterHashJoinExec1() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), + FileFragment[] job3Frags = FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); @@ -343,9 +343,9 @@ public final void testFullOuterHashJoinExec2() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), + FileFragment[] job3Frags = FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -382,9 +382,9 @@ public final void testFullOuterHashJoinExec3() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), + FileFragment[] phone3Frags = FileTablespace.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 4b8e1faf91..6804746e71 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -109,7 +109,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender1 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -148,7 +148,7 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV"); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender4 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); @@ -180,7 +180,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender2 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -220,7 +220,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender3 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -273,7 +273,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender5 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); @@ -316,9 +316,9 @@ public final void testFullOuterMergeJoin0() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] dep3Frags = - FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testFullOuterMergeJoin0"); @@ -353,9 +353,9 @@ public final void testFullOuterMergeJoin1() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] job3Frags = - FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testFullOuterMergeJoin1"); @@ -390,9 +390,9 @@ public final void testFullOuterMergeJoin2() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] job3Frags = - FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testFullOuterMergeJoin2"); @@ -428,9 +428,9 @@ public final void testFullOuterMergeJoin3() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] dep4Frags = - FileStorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testFullOuterMergeJoin3"); @@ -468,10 +468,10 @@ public final void testFullOuterMergeJoin4() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] phone3Frags = - FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), - Integer.MAX_VALUE); + FileTablespace.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), + Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testFullOuterMergeJoin4"); @@ -507,10 +507,10 @@ public final void testFullOuterMergeJoin5() throws IOException, PlanningExceptio enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] phone3Frags = - FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), - Integer.MAX_VALUE); + FileTablespace.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), + Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testFullOuterMergeJoin5"); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 624aae0871..b8f93e2a8a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -83,7 +83,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -109,7 +109,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); @@ -148,9 +148,9 @@ public void tearDown() throws Exception { @Test public final void testHashAntiJoin() throws IOException, PlanningException { - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index 36d02b177e..acaf10c329 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -85,7 +85,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); @@ -150,9 +150,9 @@ public final void testHashInnerJoin() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -193,9 +193,9 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 1a0151a0ca..b87799dc06 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -84,7 +84,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -110,7 +110,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); @@ -153,9 +153,9 @@ public void tearDown() throws Exception { @Test public final void testHashSemiJoin() throws IOException, PlanningException { - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 127d309868..4a1ca32f10 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender1 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -135,7 +135,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender2 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -175,7 +175,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender3 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -228,7 +228,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender5 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); @@ -270,9 +270,9 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), + FileFragment[] dep3Frags = FileTablespace.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); @@ -300,9 +300,9 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep @Test public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningException { - FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), + FileFragment[] job3Frags = FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); @@ -334,9 +334,9 @@ public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningExce @Test public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningException { - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), + FileFragment[] job3Frags = FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -370,9 +370,9 @@ public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningExce @Test public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningException { - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), + FileFragment[] phone3Frags = FileTablespace.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -405,9 +405,9 @@ public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningExce @Test public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningException { - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, "default.emp3", emp3.getMeta(), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, "default.emp3", emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, "default.phone3", phone3.getMeta(), + FileFragment[] phone3Frags = FileTablespace.splitNG(conf, "default.phone3", phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index c83d4366bc..5201774b09 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -85,7 +85,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -114,7 +114,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); @@ -162,9 +162,9 @@ public final void testMergeInnerJoin() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index ff9fdae52b..97cc21669d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -83,7 +83,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -107,7 +107,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); @@ -143,9 +143,9 @@ public void tearDown() throws Exception { @Test public final void testNLCrossJoin() throws IOException, PlanningException { - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -172,9 +172,9 @@ public final void testNLCrossJoin() throws IOException, PlanningException { @Test public final void testNLInnerJoin() throws IOException, PlanningException { - FileFragment[] empFrags = FileStorageManager.splitNG(conf, "default.e", employee.getMeta(), + FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileStorageManager.splitNG(conf, "default.p", people.getMeta(), + FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 8c72d39b07..7f926678d5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -80,7 +80,7 @@ public class TestPhysicalPlanner { private static SQLAnalyzer analyzer; private static LogicalPlanner planner; private static LogicalOptimizer optimizer; - private static FileStorageManager sm; + private static FileTablespace sm; private static Path testDir; private static Session session = LocalTajoTestingUtility.createDummySession(); private static QueryContext defaultContext; @@ -98,7 +98,7 @@ public static void setUp() throws Exception { util.startCatalogCluster(); conf = util.getConfiguration(); testDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestPhysicalPlanner"); - sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); + sm = (FileTablespace) TableSpaceManager.getFileStorageManager(conf); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); @@ -180,7 +180,7 @@ public static void createLargeScoreTable() throws IOException { Schema scoreSchmea = score.getSchema(); TableMeta scoreLargeMeta = CatalogUtil.newTableMeta("RAW", new KeyValueSet()); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init(); @@ -237,7 +237,7 @@ public static void tearDown() throws Exception { @Test public final void testCreateScanPlan() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCreateScanPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -268,7 +268,7 @@ public final void testCreateScanPlan() throws IOException, PlanningException { @Test public final void testCreateScanWithFilterPlan() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCreateScanWithFilterPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -297,7 +297,7 @@ public final void testCreateScanWithFilterPlan() throws IOException, PlanningExc @Test public final void testGroupByPlan() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testGroupByPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -328,7 +328,7 @@ public final void testGroupByPlan() throws IOException, PlanningException { @Test public final void testHashGroupByPlanWithALLField() throws IOException, PlanningException { // TODO - currently, this query does not use hash-based group operator. - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testHashGroupByPlanWithALLField"); @@ -358,7 +358,7 @@ public final void testHashGroupByPlanWithALLField() throws IOException, Planning @Test public final void testSortGroupByPlan() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testSortGroupByPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -421,7 +421,7 @@ public final void testSortGroupByPlan() throws IOException, PlanningException { @Test public final void testStorePlan() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -442,7 +442,7 @@ public final void testStorePlan() throws IOException, PlanningException { exec.next(); exec.close(); - Scanner scanner = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Scanner scanner = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -467,7 +467,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann TableStats stats = largeScore.getStats(); assertTrue("Checking meaningfulness of test", stats.getNumBytes() > StorageUnit.MB); - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score_large", largeScore.getMeta(), new Path(largeScore.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlanWithMaxOutputFileSize"); @@ -502,7 +502,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann // checking the file contents long totalNum = 0; for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { - Scanner scanner = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)).getFileScanner( + Scanner scanner = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)).getFileScanner( CatalogUtil.newTableMeta("CSV"), rootNode.getOutSchema(), status.getPath()); @@ -518,7 +518,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann @Test public final void testStorePlanWithRCFile() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlanWithRCFile"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -539,7 +539,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio exec.next(); exec.close(); - Scanner scanner = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)).getFileScanner( + Scanner scanner = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)).getFileScanner( outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -559,7 +559,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio @Test public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -586,7 +586,7 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc Enforcer enforcer = new Enforcer(); enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.HASH_PARTITION); - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -610,7 +610,7 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc Enforcer enforcer = new Enforcer(); enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.SORT_PARTITION); - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -626,7 +626,7 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc @Test public final void testPartitionedStorePlan() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); TaskAttemptId id = LocalTajoTestingUtility.newTaskAttemptId(masterPlan); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), id, new FileFragment[] { frags[0] }, @@ -695,7 +695,7 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, PlanningException { // Preparing working dir and input fragments - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score_large", largeScore.getMeta(), new Path(largeScore.getPath()), Integer.MAX_VALUE); TaskAttemptId id = LocalTajoTestingUtility.newTaskAttemptId(masterPlan); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testPartitionedStorePlanWithMaxFileSize"); @@ -759,7 +759,7 @@ public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, @Test public final void testPartitionedStorePlanWithEmptyGroupingSet() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); TaskAttemptId id = LocalTajoTestingUtility.newTaskAttemptId(masterPlan); @@ -826,7 +826,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() @Test public final void testAggregationFunction() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testAggregationFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -857,7 +857,7 @@ public final void testAggregationFunction() throws IOException, PlanningExceptio @Test public final void testCountFunction() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCountFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -885,7 +885,7 @@ public final void testCountFunction() throws IOException, PlanningException { @Test public final void testGroupByWithNullValue() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testGroupByWithNullValue"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -910,7 +910,7 @@ public final void testGroupByWithNullValue() throws IOException, PlanningExcepti @Test public final void testUnionPlan() throws IOException, PlanningException, CloneNotSupportedException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testUnionPlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -975,7 +975,7 @@ public final void testEvalExpr() throws IOException, PlanningException { //@Test public final void testCreateIndex() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCreateIndex"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), @@ -1002,7 +1002,7 @@ public final void testCreateIndex() throws IOException, PlanningException { @Test public final void testDuplicateEliminate() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testDuplicateEliminate"); @@ -1036,7 +1036,7 @@ public final void testDuplicateEliminate() throws IOException, PlanningException @Test public final void testSortEnforcer() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testSortEnforcer"); @@ -1087,7 +1087,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { @Test public final void testGroupByEnforcer() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testGroupByEnforcer"); Expr context = analyzer.parse(QUERIES[7]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index 94ebe51f44..600f3882ce 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -86,7 +86,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta("RAW"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); @@ -134,7 +134,7 @@ public void testExternalSortExecProgressWithPairWiseMerger() throws Exception { } private void testProgress(int sortBufferBytesNum) throws Exception { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 3455cb3f55..79db3bf7a6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -100,7 +100,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender1 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -130,7 +130,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender2 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -170,7 +170,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender3 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -229,9 +229,9 @@ public void tearDown() throws Exception { @Test public final void testRightOuter_HashJoinExec0() throws IOException, PlanningException { - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), + FileFragment[] dep3Frags = FileTablespace.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); @@ -270,9 +270,9 @@ public final void testRightOuter_HashJoinExec0() throws IOException, PlanningExc @Test public final void testRightOuter_HashJoinExec1() throws IOException, PlanningException { - FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), + FileFragment[] job3Frags = FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -311,9 +311,9 @@ public final void testRightOuter_HashJoinExec1() throws IOException, PlanningExc @Test public final void testRightOuter_HashJoinExec2() throws IOException, PlanningException { - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] job3Frags = FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), + FileFragment[] job3Frags = FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index c6bf2ef507..b67dc09826 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -107,7 +107,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender1 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -146,7 +146,7 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV"); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender4 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); @@ -178,7 +178,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender2 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -218,7 +218,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender3 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -271,7 +271,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender5 = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); @@ -314,9 +314,9 @@ public final void testRightOuterMergeJoin0() throws IOException, PlanningExcepti Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); - FileFragment[] emp3Frags = FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + FileFragment[] emp3Frags = FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] dep3Frags = FileStorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), + FileFragment[] dep3Frags = FileTablespace.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); @@ -352,9 +352,9 @@ public final void testRightOuter_MergeJoin1() throws IOException, PlanningExcept enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] job3Frags = - FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuterMergeJoin1"); @@ -388,9 +388,9 @@ public final void testRightOuterMergeJoin2() throws IOException, PlanningExcepti enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] job3Frags = - FileStorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuterMergeJoin2"); @@ -424,9 +424,9 @@ public final void testRightOuter_MergeJoin3() throws IOException, PlanningExcept enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] dep4Frags = - FileStorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuter_MergeJoin3"); @@ -461,10 +461,10 @@ public final void testRightOuter_MergeJoin4() throws IOException, PlanningExcept enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] phone3Frags = - FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), - Integer.MAX_VALUE); + FileTablespace.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), + Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testRightOuter_MergeJoin4"); @@ -498,8 +498,8 @@ public final void testRightOuterMergeJoin5() throws IOException, PlanningExcepti enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - FileStorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] phone3Frags = FileStorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), + FileTablespace.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); + FileFragment[] phone3Frags = FileTablespace.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index a350831416..6e0aa8e9ba 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -56,7 +56,7 @@ public class TestSortExec { private static SQLAnalyzer analyzer; private static LogicalPlanner planner; private static LogicalOptimizer optimizer; - private static FileStorageManager sm; + private static FileTablespace sm; private static Path workDir; private static Path tablePath; private static TableMeta employeeMeta; @@ -69,7 +69,7 @@ public static void setUp() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); - sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); + sm = (FileTablespace) TableSpaceManager.getFileStorageManager(conf); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -81,7 +81,7 @@ public static void setUp() throws Exception { tablePath = StorageUtil.concatPath(workDir, "employee", "table1"); sm.getFileSystem().mkdirs(tablePath.getParent()); - Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + Appender appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -110,7 +110,7 @@ public static void setUp() throws Exception { @Test public final void testNext() throws IOException, PlanningException { - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); + FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestSortExec"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index d2faf7e2c6..b6a47074e3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -37,7 +37,7 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.hbase.*; @@ -205,7 +205,7 @@ public void testSimpleSelectQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseTablespace) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -244,7 +244,7 @@ public void testBinaryMappedQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseTablespace) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -297,7 +297,7 @@ public void testColumnKeyValueSelectQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseTablespace) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -334,7 +334,7 @@ public void testRowFieldSelectQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseTablespace) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -469,8 +469,8 @@ private void assertIndexPredication(boolean isCompositeRowKey) throws Exception EvalNode evalNodeEq = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), new ConstEval(new TextDatum("021"))); scanNode.setQual(evalNodeEq); - StorageManager storageManager = TableSpaceManager.getStorageManager(conf, "HBASE"); - List fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + Tablespace tablespace = TableSpaceManager.getStorageManager(conf, "HBASE"); + List fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(1, fragments.size()); assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow())); assertEquals("021" + postFix, new String(((HBaseFragment)fragments.get(0)).getStopRow())); @@ -483,7 +483,7 @@ private void assertIndexPredication(boolean isCompositeRowKey) throws Exception EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); scanNode.setQual(evalNodeA); - fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(2, fragments.size()); HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); assertEquals("020", new String(fragment1.getStartRow())); @@ -498,7 +498,7 @@ private void assertIndexPredication(boolean isCompositeRowKey) throws Exception new ConstEval(new TextDatum("075"))); EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); scanNode.setQual(evalNodeB); - fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(3, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); assertEquals("020", new String(fragment1.getStartRow())); @@ -521,7 +521,7 @@ private void assertIndexPredication(boolean isCompositeRowKey) throws Exception EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); - fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(3, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); @@ -544,7 +544,7 @@ private void assertIndexPredication(boolean isCompositeRowKey) throws Exception evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); - fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); + fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(2, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index 8387abda95..3a914834cd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -569,7 +569,7 @@ private void createMultiFile(String tableName, int numRowsEachFile, TupleCreator } Path dataPath = new Path(table.getPath().toString(), fileIndex + ".csv"); fileIndex++; - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(tableMeta, schema, dataPath); appender.init(); } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 70d07c35b0..f6fd88fe28 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -57,7 +57,7 @@ public class TestResultSet { private static TajoTestingCluster util; private static TajoConf conf; private static TableDesc desc; - private static FileStorageManager sm; + private static FileTablespace sm; private static TableMeta scoreMeta; private static Schema scoreSchema; private static List serializedData; @@ -66,7 +66,7 @@ public class TestResultSet { public static void setup() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); conf = util.getConfiguration(); - sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); + sm = (FileTablespace) TableSpaceManager.getFileStorageManager(conf); scoreSchema = new Schema(); scoreSchema.addColumn("deptname", Type.TEXT); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 5efdede580..0a473b5167 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -33,7 +33,7 @@ import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; @@ -82,7 +82,7 @@ public static void setUp() throws Exception { logicalPlanner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); - StorageManager sm = TableSpaceManager.getFileStorageManager(conf); + Tablespace sm = TableSpaceManager.getFileStorageManager(conf); dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java index 52b59eadbe..3ab2df6567 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -67,8 +67,8 @@ public void test() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("ROWFILE"); - FileStorageManager sm = - (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); + FileTablespace sm = + (FileTablespace) TableSpaceManager.getFileStorageManager(conf); Path tablePath = new Path("/test"); Path metaPath = new Path(tablePath, ".meta"); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java index d007aea999..a8926a027f 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -64,7 +64,7 @@ public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List 0) { numBytes += fragmentLength; fragments.add(eachFileFragment); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpace.java deleted file mode 100644 index ef4aa9a53f..0000000000 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpace.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.hadoop.fs.Path; -import org.apache.tajo.ExecutionBlockId; -import org.apache.tajo.OverridableConf; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; -import org.apache.tajo.storage.fragment.Fragment; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; - -/** - * It manages each tablespace; e.g., HDFS, Local file system, and Amazon S3. - */ -public interface TableSpace extends Closeable { - //public void format() throws IOException; - - void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException; - - void purgeTable(TableDesc tableDesc) throws IOException; - - List getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException; - - List getSplits(String fragmentId, TableDesc tableDesc) throws IOException; - -// public void renameTable() throws IOException; -// -// public void truncateTable() throws IOException; -// -// public long availableCapacity() throws IOException; -// -// public long totalCapacity() throws IOException; - - Scanner getScanner(TableMeta meta, Schema schema, CatalogProtos.FragmentProto fragment, Schema target) throws IOException; - - Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException; - - Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException; - - Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, - LogicalPlan plan, Schema schema, - TableDesc tableDesc) throws IOException; - - void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException; - - List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException; - - void close() throws IOException; -} diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpaceManager.java index 42a5e07118..a787cdb806 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableSpaceManager.java @@ -63,10 +63,10 @@ public class TableSpaceManager { Path.class }; /** - * Cache of StorageManager. + * Cache of Tablespace. * Key is manager key(warehouse path) + store type */ - private static final Map storageManagers = Maps.newHashMap(); + private static final Map storageManagers = Maps.newHashMap(); /** * Cache of constructors for each class. Pins the classes so they * can't be garbage collected until ReflectionUtils can be collected. @@ -86,13 +86,13 @@ protected synchronized static void clearCache() { } /** - * Close StorageManager + * Close Tablespace * @throws java.io.IOException */ public static void shutdown() throws IOException { synchronized(storageManagers) { - for (StorageManager eachStorageManager: storageManagers.values()) { - eachStorageManager.close(); + for (Tablespace eachTablespace : storageManagers.values()) { + eachTablespace.close(); } } clearCache(); @@ -105,19 +105,19 @@ public static void shutdown() throws IOException { * @return * @throws IOException */ - public static StorageManager getFileStorageManager(TajoConf tajoConf) throws IOException { + public static Tablespace getFileStorageManager(TajoConf tajoConf) throws IOException { return getStorageManager(tajoConf, "CSV"); } /** - * Returns the proper StorageManager instance according to the storeType. + * Returns the proper Tablespace instance according to the storeType. * * @param tajoConf Tajo system property. * @param storeType Storage type * @return * @throws IOException */ - public static StorageManager getStorageManager(TajoConf tajoConf, String storeType) throws IOException { + public static Tablespace getStorageManager(TajoConf tajoConf, String storeType) throws IOException { FileSystem fileSystem = TajoConf.getWarehouseDir(tajoConf).getFileSystem(tajoConf); if (fileSystem != null) { return getStorageManager(tajoConf, storeType, fileSystem.getUri().toString()); @@ -127,7 +127,7 @@ public static StorageManager getStorageManager(TajoConf tajoConf, String storeTy } /** - * Returns the proper StorageManager instance according to the storeType + * Returns the proper Tablespace instance according to the storeType * * @param tajoConf Tajo system property. * @param storeType Storage type @@ -135,7 +135,7 @@ public static StorageManager getStorageManager(TajoConf tajoConf, String storeTy * @return * @throws IOException */ - private static synchronized StorageManager getStorageManager ( + private static synchronized Tablespace getStorageManager ( TajoConf tajoConf, String storeType, String managerKey) throws IOException { String typeName; @@ -147,19 +147,19 @@ private static synchronized StorageManager getStorageManager ( synchronized (storageManagers) { String storeKey = typeName + "_" + managerKey; - StorageManager manager = storageManagers.get(storeKey); + Tablespace manager = storageManagers.get(storeKey); if (manager == null) { - Class storageManagerClass = - tajoConf.getClass(String.format("tajo.storage.manager.%s.class", typeName), null, StorageManager.class); + Class storageManagerClass = + tajoConf.getClass(String.format("tajo.storage.manager.%s.class", typeName), null, Tablespace.class); if (storageManagerClass == null) { throw new IOException("Unknown Storage Type: " + typeName); } try { - Constructor constructor = - (Constructor) CONSTRUCTOR_CACHE.get(storageManagerClass); + Constructor constructor = + (Constructor) CONSTRUCTOR_CACHE.get(storageManagerClass); if (constructor == null) { constructor = storageManagerClass.getDeclaredConstructor(new Class[]{String.class}); constructor.setAccessible(true); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java similarity index 94% rename from tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java rename to tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 0751035fc8..0626da8cf5 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -42,12 +42,12 @@ import java.util.List; /** - * StorageManager manages the functions of storing and reading data. - * StorageManager is a abstract class. - * For supporting such as HDFS, HBASE, a specific StorageManager should be implemented by inheriting this class. + * Tablespace manages the functions of storing and reading data. + * Tablespace is a abstract class. + * For supporting such as HDFS, HBASE, a specific Tablespace should be implemented by inheriting this class. * */ -public abstract class StorageManager implements TableSpace { +public abstract class Tablespace { public static final PathFilter hiddenFileFilter = new PathFilter() { public boolean accept(Path p) { @@ -59,7 +59,7 @@ public boolean accept(Path p) { protected TajoConf conf; protected String storeType; - public StorageManager(String storeType) { + public Tablespace(String storeType) { this.storeType = storeType; } @@ -77,7 +77,6 @@ public StorageManager(String storeType) { * @param ifNotExists Creates the table only when the table does not exist. * @throws java.io.IOException */ - @Override public abstract void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException; /** @@ -87,7 +86,6 @@ public StorageManager(String storeType) { * @param tableDesc * @throws java.io.IOException */ - @Override public abstract void purgeTable(TableDesc tableDesc) throws IOException; /** @@ -99,7 +97,6 @@ public StorageManager(String storeType) { * @return The list of input fragments. * @throws java.io.IOException */ - @Override public abstract List getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode) throws IOException; @@ -124,7 +121,6 @@ public abstract List getNonForwardSplit(TableDesc tableDesc, int curre /** * Release storage manager resource */ - @Override public abstract void close(); @@ -162,7 +158,6 @@ public abstract TupleRange[] getInsertSortRanges(OverridableConf queryContext, T * @param node The child node of the root node. * @throws java.io.IOException */ - public abstract void rollbackOutputCommit(LogicalNode node) throws IOException; /** * Returns the current storage type. @@ -173,7 +168,7 @@ public String getStoreType() { } /** - * Initialize StorageManager instance. It should be called before using. + * Initialize Tablespace instance. It should be called before using. * * @param tajoConf * @throws java.io.IOException @@ -206,7 +201,6 @@ public List getSplits(String fragmentId, TableDesc tableDesc) throws I * @return Scanner instance * @throws java.io.IOException */ - @Override public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException { return getScanner(meta, schema, FragmentConvertor.convert(conf, fragment), target); } @@ -220,7 +214,6 @@ public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, * @return Scanner instance * @throws java.io.IOException */ - @Override public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException { return getScanner(meta, schema, fragment, schema); } @@ -235,7 +228,6 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) thro * @return Scanner instance * @throws java.io.IOException */ - @Override public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { if (fragment.isEmpty()) { Scanner scanner = new NullScanner(conf, schema, meta, fragment); @@ -326,6 +318,8 @@ public static long getFragmentLength(TajoConf conf, Fragment fragment) { } } + public abstract void rollbackOutputCommit(LogicalNode node) throws IOException; + /** * It is called after making logical plan. Storage manager should verify the schema for inserting. * @@ -333,10 +327,7 @@ public static long getFragmentLength(TajoConf conf, Fragment fragment) { * @param outSchema The output schema of select query for inserting. * @throws java.io.IOException */ - @Override - public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { - // nothing to do - } + public abstract void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException; /** * Returns the list of storage specified rewrite rules. @@ -347,11 +338,8 @@ public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throw * @return The list of storage specified rewrite rules * @throws java.io.IOException */ - @Override - public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) - throws IOException { - return null; - } + public abstract List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) + throws IOException; /** * Finalizes result data. Tajo stores result data in the staging directory. @@ -366,7 +354,6 @@ public List getRewriteRules(OverridableConf queryContext * @return Saved path * @throws java.io.IOException */ - @Override public abstract Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException; diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index aa078a7494..93611fbd70 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -23,11 +23,11 @@ tajo.storage.manager.hdfs.class - org.apache.tajo.storage.FileStorageManager + org.apache.tajo.storage.FileTablespace tajo.storage.manager.hbase.class - org.apache.tajo.storage.hbase.HBaseStorageManager + org.apache.tajo.storage.hbase.HBaseTablespace diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml index 712f66428f..6aa32fc0da 100644 --- a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml @@ -28,11 +28,11 @@ tajo.storage.manager.hdfs.class - org.apache.tajo.storage.FileStorageManager + org.apache.tajo.storage.FileTablespace tajo.storage.manager.hbase.class - org.apache.tajo.storage.hbase.HBaseStorageManager + org.apache.tajo.storage.hbase.HBaseTablespace diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java index 09a86b4e21..b1a2d592c5 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java @@ -46,8 +46,8 @@ public HBasePutAppender(Configuration conf, TaskAttemptId taskAttemptId, public void init() throws IOException { super.init(); - Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(conf, meta); - HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager((TajoConf) conf, "HBASE")) + Configuration hbaseConf = HBaseTablespace.getHBaseConfiguration(conf, meta); + HConnection hconn = ((HBaseTablespace) TableSpaceManager.getStorageManager((TajoConf) conf, "HBASE")) .getConnection(hbaseConf); htable = hconn.getTable(columnMapping.getHbaseTableName()); htable.setAutoFlushTo(false); diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index 24bfd4d283..992c13cdd3 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -133,7 +133,7 @@ public void init() throws IOException { rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); - hbaseConf = HBaseStorageManager.getHBaseConfiguration(conf, meta); + hbaseConf = HBaseTablespace.getHBaseConfiguration(conf, meta); initScanner(); } @@ -181,7 +181,7 @@ private void initScanner() throws IOException { } if (htable == null) { - HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseTablespace) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(hbaseConf); htable = hconn.getTable(fragment.getHbaseTableName()); } diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java similarity index 98% rename from tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java rename to tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index 3653574bca..670f87e9a8 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -43,7 +43,6 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.exception.UnimplementedException; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.CreateTableNode; @@ -64,14 +63,14 @@ import java.util.*; /** - * StorageManager for HBase table. + * Tablespace for HBase table. */ -public class HBaseStorageManager extends StorageManager { - private static final Log LOG = LogFactory.getLog(HBaseStorageManager.class); +public class HBaseTablespace extends Tablespace { + private static final Log LOG = LogFactory.getLog(HBaseTablespace.class); private Map connMap = new HashMap(); - public HBaseStorageManager (String storeType) { + public HBaseTablespace(String storeType) { super(storeType); } @@ -949,7 +948,7 @@ public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId fina Path stagingDir = new Path(queryContext.get(QueryVars.STAGING_DIR)); Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); - Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); + Configuration hbaseConf = HBaseTablespace.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); hbaseConf.set("hbase.loadincremental.threads.max", "2"); JobContextImpl jobContext = new JobContextImpl(hbaseConf, @@ -995,7 +994,7 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc } ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta()); - Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); + Configuration hbaseConf = HBaseTablespace.getHBaseConfiguration(queryContext.getConf(), tableDesc.getMeta()); HTable htable = new HTable(hbaseConf, columnMapping.getHbaseTableName()); try { diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java index 39ccf44fdd..b59fe7b1fa 100644 --- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java @@ -46,8 +46,8 @@ public void testGetIndexPredications() throws Exception { EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); scanNode.setQual(evalNodeA); - HBaseStorageManager storageManager = - (HBaseStorageManager) TableSpaceManager.getStorageManager(new TajoConf(), "HBASE"); + HBaseTablespace storageManager = + (HBaseTablespace) TableSpaceManager.getStorageManager(new TajoConf(), "HBASE"); List> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn}); assertNotNull(indexEvals); assertEquals(1, indexEvals.size()); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java index c04177124f..5c8242f6c7 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java @@ -57,7 +57,7 @@ public FileAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema sche throw new IllegalArgumentException("Configuration must be an instance of TajoConf"); } - this.path = ((FileStorageManager) TableSpaceManager.getFileStorageManager((TajoConf) conf)) + this.path = ((FileTablespace) TableSpaceManager.getFileStorageManager((TajoConf) conf)) .getAppenderFilePath(taskAttemptId, workDir); } else { this.path = workDir; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java similarity index 95% rename from tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java rename to tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index 4efc3b73da..6ab8574b99 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -35,6 +35,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Bytes; @@ -45,8 +46,8 @@ import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -public class FileStorageManager extends StorageManager { - private final Log LOG = LogFactory.getLog(FileStorageManager.class); +public class FileTablespace extends Tablespace { + private final Log LOG = LogFactory.getLog(FileTablespace.class); static final String OUTPUT_FILE_PREFIX="part-"; static final ThreadLocal OUTPUT_FILE_FORMAT_STAGE = @@ -86,7 +87,7 @@ public NumberFormat initialValue() { protected boolean blocksMetadataEnabled; private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); - public FileStorageManager(String storeType) { + public FileTablespace(String storeType) { super(storeType); } @@ -117,10 +118,6 @@ public FileSystem getFileSystem() { return this.fs; } - public Path getWarehouseDir() { - return this.tableBaseDir; - } - public void delete(Path tablePath) throws IOException { FileSystem fs = tablePath.getFileSystem(conf); fs.delete(tablePath, true); @@ -131,20 +128,6 @@ public boolean exists(Path path) throws IOException { return fileSystem.exists(path); } - /** - * This method deletes only data contained in the given path. - * - * @param path The path in which data are deleted. - * @throws IOException - */ - public void deleteData(Path path) throws IOException { - FileSystem fileSystem = path.getFileSystem(conf); - FileStatus[] fileLists = fileSystem.listStatus(path); - for (FileStatus status : fileLists) { - fileSystem.delete(status.getPath(), true); - } - } - public Path getTablePath(String tableName) { return new Path(tableBaseDir, tableName); } @@ -180,23 +163,6 @@ public FileFragment[] split(String tableName, long fragmentSize) throws IOExcept return split(tableName, tablePath, fragmentSize); } - public FileFragment[] splitBroadcastTable(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - List listTablets = new ArrayList(); - FileFragment tablet; - - FileStatus[] fileLists = fs.listStatus(tablePath); - for (FileStatus file : fileLists) { - tablet = new FileFragment(tablePath.getName(), file.getPath(), 0, file.getLen()); - listTablets.add(tablet); - } - - FileFragment[] tablets = new FileFragment[listTablets.size()]; - listTablets.toArray(tablets); - - return tablets; - } - public FileFragment[] split(Path tablePath) throws IOException { FileSystem fs = tablePath.getFileSystem(conf); return split(tablePath.getName(), tablePath, fs.getDefaultBlockSize()); @@ -802,7 +768,7 @@ private void getNonZeroLengthDataFiles(FileSystem fs, Path path, List 0) { @@ -876,6 +842,16 @@ public void beforeInsertOrCATS(LogicalNode node) throws IOException { public void rollbackOutputCommit(LogicalNode node) throws IOException { } + @Override + public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throws IOException { + } + + @Override + public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) + throws IOException { + return null; + } + @Override public Path commitOutputData(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { @@ -889,26 +865,6 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc return null; } - /** - * Returns Scanner instance. - * - * @param conf The system property - * @param meta The table meta - * @param schema The input schema - * @param path The data file path - * @return Scanner instance - * @throws java.io.IOException - */ - public static synchronized SeekableScanner getSeekableScanner( - TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { - - FileSystem fs = path.getFileSystem(conf); - FileStatus status = fs.getFileStatus(path); - FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); - - return TableSpaceManager.getSeekableScanner(conf, meta, schema, fragment, schema); - } - /** * Finalizes result data. Tajo stores result data in the staging directory. * If the query fails, clean up the staging directory. @@ -993,7 +949,7 @@ protected Path commitOutputData(OverridableConf queryContext, boolean changeFile if (fs.exists(finalOutputDir)) { fs.mkdirs(oldTableDir); - for (FileStatus status : fs.listStatus(finalOutputDir, StorageManager.hiddenFileFilter)) { + for (FileStatus status : fs.listStatus(finalOutputDir, Tablespace.hiddenFileFilter)) { fs.rename(status.getPath(), oldTableDir); } @@ -1015,7 +971,7 @@ protected Path commitOutputData(OverridableConf queryContext, boolean changeFile if (movedToOldTable && !committed) { // if commit is failed, recover the old data - for (FileStatus status : fs.listStatus(finalOutputDir, StorageManager.hiddenFileFilter)) { + for (FileStatus status : fs.listStatus(finalOutputDir, Tablespace.hiddenFileFilter)) { fs.delete(status.getPath(), true); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index 1846ed6b36..66c7f134b9 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -82,7 +82,7 @@ public HashShuffleAppender getAppender(TajoConf tajoConf, ExecutionBlockId ebId, if (!fs.exists(dataFile.getParent())) { fs.mkdirs(dataFile.getParent()); } - FileAppender appender = (FileAppender)((FileStorageManager) TableSpaceManager.getFileStorageManager(tajoConf)) + FileAppender appender = (FileAppender)((FileTablespace) TableSpaceManager.getFileStorageManager(tajoConf)) .getAppender(meta, outSchema, dataFile); appender.enableStats(); appender.init(); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index 779f908946..75ad0d58d8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -122,7 +122,7 @@ private void storageCompressionTest(String storeType, Class tajo.storage.manager.hdfs.class - org.apache.tajo.storage.FileStorageManager + org.apache.tajo.storage.FileTablespace tajo.storage.manager.hbase.class - org.apache.tajo.storage.hbase.HBaseStorageManager + org.apache.tajo.storage.hbase.HBaseTablespace From aed6daab33190f9b8675c2ba97d346ed107c28e2 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 18 May 2015 20:32:25 -0700 Subject: [PATCH 2/2] Fixed missed symbol. --- .../test/java/org/apache/tajo/engine/query/TestJoinQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 0e42412601..a65c16597b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -264,7 +264,7 @@ protected static void createMultiFile(String tableName, int numRowsEachFile, Tup } Path dataPath = new Path(table.getPath().toString(), fileIndex + ".csv"); fileIndex++; - appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) + appender = ((FileTablespace) TableSpaceManager.getFileStorageManager(conf)) .getAppender(tableMeta, schema, dataPath); appender.init(); }