From d4d3263f3d64762591caf9ec942064b28475f728 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 21 Jul 2015 15:04:26 +0900 Subject: [PATCH 1/3] TAJO-1496 --- .../org/apache/tajo/catalog/CatalogUtil.java | 2 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 6 +- .../src/main/resources/storage-default.xml | 18 +- .../src/test/resources/storage-default.xml | 63 +- .../java/org/apache/tajo/storage/CSVFile.java | 589 ------------------ .../storage/CompressedSplitLineReader.java | 182 ------ .../org/apache/tajo/storage/LineReader.java | 559 ----------------- .../apache/tajo/storage/SplitLineReader.java | 39 -- .../tajo/storage/TestCompressionStorages.java | 4 +- .../apache/tajo/storage/TestMergeScanner.java | 3 +- .../index/TestSingleCSVFileBSTIndex.java | 9 +- .../src/test/resources/storage-default.xml | 18 +- 12 files changed, 57 insertions(+), 1435 deletions(-) delete mode 100644 tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java delete mode 100644 tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java delete mode 100644 tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/LineReader.java delete mode 100644 tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/SplitLineReader.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index ecf8890a22..c0fca14f0e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -287,7 +287,7 @@ public static String getStoreTypeString(final StoreType type) { public static StoreType getStoreType(final String typeStr) { if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) { - return StoreType.CSV; + return StoreType.TEXTFILE; } else if (typeStr.equalsIgnoreCase(StoreType.RAW.name())) { return StoreType.RAW; } else if (typeStr.equalsIgnoreCase(StoreType.ROWFILE.name())) { diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 5005670045..c96311ee56 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -398,13 +398,15 @@ public void testTimeZoneTest1() throws Exception { @Test public void testTimeZoneTest2() throws Exception { String tableName = "test1"; - tajoCli.executeMetaCommand("\\set TIMEZONE GMT+1"); + tajoCli.executeMetaCommand("\\set TIMEZONE GMT+0"); tajoCli.executeScript("create table " + tableName + " (col1 TIMESTAMP)"); tajoCli.executeScript("insert into " + tableName + " select to_timestamp(0)"); + tajoCli.executeMetaCommand("\\set TIMEZONE GMT+1"); tajoCli.executeScript("select * from " + tableName); String consoleResult = new String(out.toByteArray()); tajoCli.executeScript("DROP TABLE " + tableName + " PURGE"); - assertTrue(consoleResult.contains("1970-01-01 01:00:00")); + System.out.println(consoleResult); + assertTrue(consoleResult.contains("1970-01-01 00:00:00")); } @Test(timeout = 3000) diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml index 93611fbd70..09261a9850 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml @@ -39,7 +39,7 @@ tajo.storage.scanner-handler - text,csv,json,raw,rcfile,row,parquet,sequencefile,avro,hbase + text,json,raw,rcfile,row,parquet,sequencefile,avro,hbase @@ -47,10 +47,6 @@ tajo.storage.fragment.text.class org.apache.tajo.storage.fragment.FileFragment - - tajo.storage.fragment.csv.class - org.apache.tajo.storage.fragment.FileFragment - tajo.storage.fragment.json.class org.apache.tajo.storage.fragment.FileFragment @@ -90,11 +86,6 @@ org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner - - tajo.storage.scanner-handler.csv.class - org.apache.tajo.storage.CSVFile$CSVScanner - - tajo.storage.scanner-handler.json.class org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner @@ -138,7 +129,7 @@ tajo.storage.appender-handler - text,csv,raw,rcfile,row,parquet,sequencefile,avro,hbase + text,raw,rcfile,row,parquet,sequencefile,avro,hbase @@ -146,11 +137,6 @@ org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender - - tajo.storage.appender-handler.csv.class - org.apache.tajo.storage.CSVFile$CSVAppender - - tajo.storage.appender-handler.json.class org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml index 6aa32fc0da..ba7f4e820f 100644 --- a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml @@ -38,33 +38,33 @@ tajo.storage.scanner-handler - csv,raw,rcfile,row,trevni,parquet,sequencefile,avro + text,json,raw,rcfile,row,parquet,sequencefile,avro,hbase - tajo.storage.fragment.csv.class + tajo.storage.fragment.text.class org.apache.tajo.storage.fragment.FileFragment - tajo.storage.fragment.raw.class + tajo.storage.fragment.json.class org.apache.tajo.storage.fragment.FileFragment - tajo.storage.fragment.rcfile.class + tajo.storage.fragment.raw.class org.apache.tajo.storage.fragment.FileFragment - tajo.storage.fragment.row.class + tajo.storage.fragment.rcfile.class org.apache.tajo.storage.fragment.FileFragment - tajo.storage.fragment.trevni.class + tajo.storage.fragment.row.class org.apache.tajo.storage.fragment.FileFragment tajo.storage.fragment.parquet.class - org.apache.tajo.storage.FileFragment + org.apache.tajo.storage.fragment.FileFragment tajo.storage.fragment.sequencefile.class @@ -74,11 +74,20 @@ tajo.storage.fragment.avro.class org.apache.tajo.storage.fragment.FileFragment + + tajo.storage.fragment.hbase.class + org.apache.tajo.storage.hbase.HBaseFragment + - tajo.storage.scanner-handler.csv.class - org.apache.tajo.storage.CSVFile$CSVScanner + tajo.storage.scanner-handler.text.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner + + + + tajo.storage.scanner-handler.json.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner @@ -96,11 +105,6 @@ org.apache.tajo.storage.RowFile$RowFileScanner - - tajo.storage.scanner-handler.trevni.class - org.apache.tajo.storage.trevni.TrevniScanner - - tajo.storage.scanner-handler.parquet.class org.apache.tajo.storage.parquet.ParquetScanner @@ -116,15 +120,25 @@ org.apache.tajo.storage.avro.AvroScanner + + tajo.storage.scanner-handler.hbase.class + org.apache.tajo.storage.hbase.HBaseScanner + + tajo.storage.appender-handler - csv,raw,rcfile,row,trevni,parquet,sequencefile,avro + text,raw,rcfile,row,parquet,sequencefile,avro,hbase + + + + tajo.storage.appender-handler.text.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender - tajo.storage.appender-handler.csv.class - org.apache.tajo.storage.CSVFile$CSVAppender + tajo.storage.appender-handler.json.class + org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender @@ -142,11 +156,6 @@ org.apache.tajo.storage.RowFile$RowFileAppender - - tajo.storage.appender-handler.trevni.class - org.apache.tajo.storage.trevni.TrevniAppender - - tajo.storage.appender-handler.parquet.class org.apache.tajo.storage.parquet.ParquetAppender @@ -162,6 +171,16 @@ org.apache.tajo.storage.avro.AvroAppender + + tajo.storage.appender-handler.hbase.class + org.apache.tajo.storage.hbase.HFileAppender + + + + tajo.storage.appender-handler.hfile.class + org.apache.tajo.storage.hbase.HFileAppender + + tajo.storage.text.io.read-buffer.bytes diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java deleted file mode 100644 index 8b8ca76da9..0000000000 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java +++ /dev/null @@ -1,589 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.compress.*; -import org.apache.tajo.TaskAttemptId; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.exception.UnsupportedException; -import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.storage.compress.CodecPool; -import org.apache.tajo.storage.exception.AlreadyExistsStorageException; -import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream; -import org.apache.tajo.util.Bytes; -import org.apache.tajo.util.BytesUtils; - -import java.io.*; -import java.util.ArrayList; -import java.util.Arrays; - -public class CSVFile { - - public static final byte LF = '\n'; - public static int EOF = -1; - - private static final Log LOG = LogFactory.getLog(CSVFile.class); - - public static class CSVAppender extends FileAppender { - private final TableMeta meta; - private final Schema schema; - private final int columnNum; - private final FileSystem fs; - private FSDataOutputStream fos; - private DataOutputStream outputStream; - private CompressionOutputStream deflateFilter; - private byte[] delimiter; - private TableStatistics stats = null; - private Compressor compressor; - private CompressionCodecFactory codecFactory; - private CompressionCodec codec; - private Path compressedPath; - private byte[] nullChars; - private int BUFFER_SIZE = 128 * 1024; - private int bufferedBytes = 0; - private long pos = 0; - private boolean isShuffle; - - private NonSyncByteArrayOutputStream os = new NonSyncByteArrayOutputStream(BUFFER_SIZE); - private SerializerDeserializer serde; - - public CSVAppender(Configuration conf, final TaskAttemptId taskAttemptId, - final Schema schema, final TableMeta meta, final Path workDir) throws IOException { - super(conf, taskAttemptId, schema, meta, workDir); - this.fs = workDir.getFileSystem(conf); - this.meta = meta; - this.schema = schema; - this.delimiter = StringEscapeUtils.unescapeJava( - this.meta.getOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER)) - .getBytes(Bytes.UTF8_CHARSET); - - this.columnNum = schema.size(); - - String nullCharacters = StringEscapeUtils.unescapeJava( - this.meta.getOption(StorageConstants.TEXT_NULL, NullDatum.DEFAULT_TEXT)); - - if (StringUtils.isEmpty(nullCharacters)) { - nullChars = NullDatum.get().asTextBytes(); - } else { - nullChars = nullCharacters.getBytes(Bytes.UTF8_CHARSET); - } - } - - @Override - public void init() throws IOException { - if (!fs.exists(path.getParent())) { - throw new FileNotFoundException(path.getParent().toString()); - } - - //determine the intermediate file type - String store = conf.get(TajoConf.ConfVars.SHUFFLE_FILE_FORMAT.varname, - TajoConf.ConfVars.SHUFFLE_FILE_FORMAT.defaultVal); - if (enabledStats && CatalogProtos.StoreType.CSV == CatalogProtos.StoreType.valueOf(store.toUpperCase())) { - isShuffle = true; - } else { - isShuffle = false; - } - - if(this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) { - String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); - codecFactory = new CompressionCodecFactory(conf); - codec = codecFactory.getCodecByClassName(codecName); - compressor = CodecPool.getCompressor(codec); - if(compressor != null) compressor.reset(); //builtin gzip is null - - String extension = codec.getDefaultExtension(); - compressedPath = path.suffix(extension); - - if (fs.exists(compressedPath)) { - throw new AlreadyExistsStorageException(compressedPath); - } - - fos = fs.create(compressedPath); - deflateFilter = codec.createOutputStream(fos, compressor); - outputStream = new DataOutputStream(deflateFilter); - - } else { - if (fs.exists(path)) { - throw new AlreadyExistsStorageException(path); - } - fos = fs.create(path); - outputStream = new DataOutputStream(new BufferedOutputStream(fos)); - } - - if (enabledStats) { - this.stats = new TableStatistics(this.schema); - } - - try { - //It will be remove, because we will add custom serde in textfile - String serdeClass = this.meta.getOption(StorageConstants.CSVFILE_SERDE, - TextSerializerDeserializer.class.getName()); - serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); - serde.init(schema); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new IOException(e); - } - - os.reset(); - pos = fos.getPos(); - bufferedBytes = 0; - super.init(); - } - - - @Override - public void addTuple(Tuple tuple) throws IOException { - int rowBytes = 0; - - for (int i = 0; i < columnNum; i++) { - rowBytes += serde.serialize(i, tuple, os, nullChars); - - if(columnNum - 1 > i){ - os.write(delimiter); - rowBytes += delimiter.length; - } - if (isShuffle) { - // it is to calculate min/max values, and it is only used for the intermediate file. - stats.analyzeField(i, tuple); - } - } - os.write(LF); - rowBytes += 1; - - pos += rowBytes; - bufferedBytes += rowBytes; - if(bufferedBytes > BUFFER_SIZE){ - flushBuffer(); - } - // Statistical section - if (enabledStats) { - stats.incrementRow(); - } - } - - private void flushBuffer() throws IOException { - if(os.getLength() > 0) { - os.writeTo(outputStream); - os.reset(); - bufferedBytes = 0; - } - } - @Override - public long getOffset() throws IOException { - return pos; - } - - @Override - public void flush() throws IOException { - flushBuffer(); - outputStream.flush(); - } - - @Override - public void close() throws IOException { - - try { - flush(); - - // Statistical section - if (enabledStats) { - stats.setNumBytes(getOffset()); - } - - if(deflateFilter != null) { - deflateFilter.finish(); - deflateFilter.resetState(); - deflateFilter = null; - } - } finally { - IOUtils.cleanup(LOG, os, fos); - if (compressor != null) { - CodecPool.returnCompressor(compressor); - compressor = null; - } - } - } - - @Override - public TableStats getStats() { - if (enabledStats) { - return stats.getTableStat(); - } else { - return null; - } - } - - public boolean isCompress() { - return compressor != null; - } - - public String getExtension() { - return codec != null ? codec.getDefaultExtension() : ""; - } - } - - public static class CSVScanner extends FileScanner implements SeekableScanner { - public CSVScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) - throws IOException { - super(conf, schema, meta, fragment); - factory = new CompressionCodecFactory(conf); - codec = factory.getCodec(this.fragment.getPath()); - if (codec == null || codec instanceof SplittableCompressionCodec) { - splittable = true; - } - - //Delimiter - this.delimiter = StringEscapeUtils.unescapeJava( - meta.getOption(StorageConstants.TEXT_DELIMITER, - meta.getOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER))) - .getBytes(Bytes.UTF8_CHARSET); - - String nullCharacters = StringEscapeUtils.unescapeJava( - meta.getOption(StorageConstants.TEXT_NULL, - meta.getOption(StorageConstants.CSVFILE_NULL, NullDatum.DEFAULT_TEXT))); - - if (StringUtils.isEmpty(nullCharacters)) { - nullChars = NullDatum.get().asTextBytes(); - } else { - nullChars = nullCharacters.getBytes(Bytes.UTF8_CHARSET); - } - } - - private final static int DEFAULT_PAGE_SIZE = 256 * 1024; - private byte[] delimiter; - private FileSystem fs; - private FSDataInputStream fis; - private InputStream is; //decompressd stream - private CompressionCodecFactory factory; - private CompressionCodec codec; - private Decompressor decompressor; - private Seekable filePosition; - private boolean splittable = false; - private long startOffset, end, pos; - private int currentIdx = 0, validIdx = 0, recordCount = 0; - private int[] targetColumnIndexes; - private boolean eof = false; - private final byte[] nullChars; - private SplitLineReader reader; - private ArrayList fileOffsets; - private ArrayList rowLengthList; - private ArrayList startOffsets; - private NonSyncByteArrayOutputStream buffer; - private SerializerDeserializer serde; - - @Override - public void init() throws IOException { - fileOffsets = new ArrayList(); - rowLengthList = new ArrayList(); - startOffsets = new ArrayList(); - buffer = new NonSyncByteArrayOutputStream(DEFAULT_PAGE_SIZE); - - // FileFragment information - if(fs == null) { - fs = FileScanner.getFileSystem((TajoConf)conf, fragment.getPath()); - } - if(fis == null) fis = fs.open(fragment.getPath()); - - recordCount = 0; - pos = startOffset = fragment.getStartKey(); - end = startOffset + fragment.getLength(); - - if (codec != null) { - decompressor = CodecPool.getDecompressor(codec); - if (codec instanceof SplittableCompressionCodec) { - SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream( - fis, decompressor, startOffset, end, - SplittableCompressionCodec.READ_MODE.BYBLOCK); - - reader = new CompressedSplitLineReader(cIn, conf, null); - startOffset = cIn.getAdjustedStart(); - end = cIn.getAdjustedEnd(); - filePosition = cIn; - is = cIn; - } else { - is = new DataInputStream(codec.createInputStream(fis, decompressor)); - reader = new SplitLineReader(is, null); - filePosition = fis; - } - } else { - fis.seek(startOffset); - filePosition = fis; - is = fis; - reader = new SplitLineReader(is, null); - } - - if (targets == null) { - targets = schema.toArray(); - } - - targetColumnIndexes = new int[targets.length]; - for (int i = 0; i < targets.length; i++) { - targetColumnIndexes[i] = schema.getColumnId(targets[i].getQualifiedName()); - } - - try { - //FIXME - String serdeClass = this.meta.getOption(StorageConstants.CSVFILE_SERDE, - TextSerializerDeserializer.class.getName()); - serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); - serde.init(schema); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new IOException(e); - } - - super.init(); - Arrays.sort(targetColumnIndexes); - if (LOG.isDebugEnabled()) { - LOG.debug("CSVScanner open:" + fragment.getPath() + "," + startOffset + "," + end + - "," + fs.getFileStatus(fragment.getPath()).getLen()); - } - - if (startOffset != 0) { - pos += reader.readLine(new Text(), 0, maxBytesToConsume(pos)); - } - eof = false; - page(); - } - - private int maxBytesToConsume(long pos) { - return isCompress() ? Integer.MAX_VALUE : (int) Math.min(Integer.MAX_VALUE, end - pos); - } - - private long fragmentable() throws IOException { - return end - getFilePosition(); - } - - private long getFilePosition() throws IOException { - long retVal; - if (isCompress()) { - retVal = filePosition.getPos(); - } else { - retVal = pos; - } - return retVal; - } - - private void page() throws IOException { -// // Index initialization - currentIdx = 0; - validIdx = 0; - int currentBufferPos = 0; - int bufferedSize = 0; - - buffer.reset(); - startOffsets.clear(); - rowLengthList.clear(); - fileOffsets.clear(); - - if(eof) { - return; - } - - while (DEFAULT_PAGE_SIZE >= bufferedSize){ - - int ret = reader.readDefaultLine(buffer, rowLengthList, Integer.MAX_VALUE, Integer.MAX_VALUE); - - if(ret == 0){ - break; - } else { - fileOffsets.add(pos); - pos += ret; - startOffsets.add(currentBufferPos); - currentBufferPos += rowLengthList.get(rowLengthList.size() - 1); - bufferedSize += ret; - validIdx++; - recordCount++; - } - - if(getFilePosition() > end && !reader.needAdditionalRecordAfterSplit()){ - eof = true; - break; - } - } - if (tableStats != null) { - tableStats.setReadBytes(pos - startOffset); - tableStats.setNumRows(recordCount); - } - } - - @Override - public float getProgress() { - try { - if(eof) { - return 1.0f; - } - long filePos = getFilePosition(); - if (startOffset == filePos) { - return 0.0f; - } else { - long readBytes = filePos - startOffset; - long remainingBytes = Math.max(end - filePos, 0); - return Math.min(1.0f, (float)(readBytes) / (float)(readBytes + remainingBytes)); - } - } catch (IOException e) { - LOG.error(e.getMessage(), e); - return 0.0f; - } - } - - @Override - public Tuple next() throws IOException { - try { - if (currentIdx == validIdx) { - if (eof) { - return null; - } else { - page(); - - if(currentIdx == validIdx){ - return null; - } - } - } - - long offset = -1; - if(!isCompress()){ - offset = fileOffsets.get(currentIdx); - } - - byte[][] cells = BytesUtils.splitPreserveAllTokens(buffer.getData(), startOffsets.get(currentIdx), - rowLengthList.get(currentIdx), delimiter, targetColumnIndexes, schema.size()); - currentIdx++; - return new LazyTuple(schema, cells, offset, nullChars, serde); - } catch (Throwable t) { - LOG.error("Tuple list length: " + (fileOffsets != null ? fileOffsets.size() : 0), t); - LOG.error("Tuple list current index: " + currentIdx, t); - throw new IOException(t); - } - } - - private boolean isCompress() { - return codec != null; - } - - @Override - public void reset() throws IOException { - if (decompressor != null) { - CodecPool.returnDecompressor(decompressor); - decompressor = null; - } - - init(); - } - - @Override - public void close() throws IOException { - try { - if (tableStats != null) { - tableStats.setReadBytes(pos - startOffset); //Actual Processed Bytes. (decompressed bytes + overhead) - tableStats.setNumRows(recordCount); - } - - IOUtils.cleanup(LOG, reader, is, fis); - fs = null; - is = null; - fis = null; - if (LOG.isDebugEnabled()) { - LOG.debug("CSVScanner processed record:" + recordCount); - } - } finally { - if (decompressor != null) { - CodecPool.returnDecompressor(decompressor); - decompressor = null; - } - } - } - - @Override - public boolean isProjectable() { - return false; - } - - @Override - public boolean isSelectable() { - return false; - } - - @Override - public void setFilter(EvalNode filter) { - throw new UnsupportedException(); - } - - @Override - public void seek(long offset) throws IOException { - if(isCompress()) throw new UnsupportedException(); - - int tupleIndex = Arrays.binarySearch(fileOffsets.toArray(), offset); - - if (tupleIndex > -1) { - this.currentIdx = tupleIndex; - } else if (isSplittable() && end >= offset || startOffset <= offset) { - eof = false; - fis.seek(offset); - pos = offset; - reader.reset(); - this.currentIdx = 0; - this.validIdx = 0; - // pageBuffer(); - } else { - throw new IOException("invalid offset " + - " < start : " + startOffset + " , " + - " end : " + end + " , " + - " filePos : " + filePosition.getPos() + " , " + - " input offset : " + offset + " >"); - } - } - - @Override - public long getNextOffset() throws IOException { - if(isCompress()) throw new UnsupportedException(); - - if (this.currentIdx == this.validIdx) { - if (fragmentable() <= 0) { - return -1; - } else { - page(); - if(currentIdx == validIdx) return -1; - } - } - return fileOffsets.get(currentIdx); - } - - @Override - public boolean isSplittable(){ - return splittable; - } - } -} diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java deleted file mode 100644 index 4f58e68e63..0000000000 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CompressedSplitLineReader.java +++ /dev/null @@ -1,182 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.compress.SplitCompressionInputStream; -import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; - -/** - * Line reader for compressed splits - * - * Reading records from a compressed split is tricky, as the - * LineRecordReader is using the reported compressed input stream - * position directly to determine when a split has ended. In addition the - * compressed input stream is usually faking the actual byte position, often - * updating it only after the first compressed block after the split is - * accessed. - * - * Depending upon where the last compressed block of the split ends relative - * to the record delimiters it can be easy to accidentally drop the last - * record or duplicate the last record between this split and the next. - * - * Split end scenarios: - * - * 1) Last block of split ends in the middle of a record - * Nothing special that needs to be done here, since the compressed input - * stream will report a position after the split end once the record - * is fully read. The consumer of the next split will discard the - * partial record at the start of the split normally, and no data is lost - * or duplicated between the splits. - * - * 2) Last block of split ends in the middle of a delimiter - * The line reader will continue to consume bytes into the next block to - * locate the end of the delimiter. If a custom delimiter is being used - * then the next record must be read by this split or it will be dropped. - * The consumer of the next split will not recognize the partial - * delimiter at the beginning of its split and will discard it along with - * the next record. - * - * However for the default delimiter processing there is a special case - * because CR, LF, and CRLF are all valid record delimiters. If the - * block ends with a CR then the reader must peek at the next byte to see - * if it is an LF and therefore part of the same record delimiter. - * Peeking at the next byte is an access to the next block and triggers - * the stream to report the end of the split. There are two cases based - * on the next byte: - * - * A) The next byte is LF - * The split needs to end after the current record is returned. The - * consumer of the next split will discard the first record, which - * is degenerate since LF is itself a delimiter, and start consuming - * records after that byte. If the current split tries to read - * another record then the record will be duplicated between splits. - * - * B) The next byte is not LF - * The current record will be returned but the stream will report - * the split has ended due to the peek into the next block. If the - * next record is not read then it will be lost, as the consumer of - * the next split will discard it before processing subsequent - * records. Therefore the next record beyond the reported split end - * must be consumed by this split to avoid data loss. - * - * 3) Last block of split ends at the beginning of a delimiter - * This is equivalent to case 1, as the reader will consume bytes into - * the next block and trigger the end of the split. No further records - * should be read as the consumer of the next split will discard the - * (degenerate) record at the beginning of its split. - * - * 4) Last block of split ends at the end of a delimiter - * Nothing special needs to be done here. The reader will not start - * examining the bytes into the next block until the next record is read, - * so the stream will not report the end of the split just yet. Once the - * next record is read then the next block will be accessed and the - * stream will indicate the end of the split. The consumer of the next - * split will correctly discard the first record of its split, and no - * data is lost or duplicated. - * - * If the default delimiter is used and the block ends at a CR then this - * is treated as case 2 since the reader does not yet know without - * looking at subsequent bytes whether the delimiter has ended. - * - * NOTE: It is assumed that compressed input streams *never* return bytes from - * multiple compressed blocks from a single read. Failure to do so will - * violate the buffering performed by this class, as it will access - * bytes into the next block after the split before returning all of the - * records from the previous block. - */ - -public class CompressedSplitLineReader extends SplitLineReader { - SplitCompressionInputStream scin; - private boolean usingCRLF; - private boolean needAdditionalRecord = false; - private boolean finished = false; - - public CompressedSplitLineReader(SplitCompressionInputStream in, - Configuration conf, - byte[] recordDelimiterBytes) - throws IOException { - super(in, conf, recordDelimiterBytes); - scin = in; - usingCRLF = (recordDelimiterBytes == null); - } - - @Override - protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter) - throws IOException { - int bytesRead = in.read(buffer); - - // If the split ended in the middle of a record delimiter then we need - // to read one additional record, as the consumer of the next split will - // not recognize the partial delimiter as a record. - // However if using the default delimiter and the next character is a - // linefeed then next split will treat it as a delimiter all by itself - // and the additional record read should not be performed. - if (inDelimiter && bytesRead > 0) { - if (usingCRLF) { - needAdditionalRecord = (buffer[0] != '\n'); - } else { - needAdditionalRecord = true; - } - } - return bytesRead; - } - - @Override - public int readLine(Text str, int maxLineLength, int maxBytesToConsume) - throws IOException { - int bytesRead = 0; - if (!finished) { - // only allow at most one more record to be read after the stream - // reports the split ended - if (scin.getPos() > scin.getAdjustedEnd()) { - finished = true; - } - - bytesRead = super.readLine(str, maxLineLength, maxBytesToConsume); - } - return bytesRead; - } - - @Override - public int readDefaultLine(NonSyncByteArrayOutputStream str, ArrayList offsets, int maxLineLength - , int maxBytesToConsume) throws IOException { - int bytesRead = 0; - if (!finished) { - // only allow at most one more record to be read after the stream - // reports the split ended - if (scin.getPos() > scin.getAdjustedEnd()) { - finished = true; - } - - bytesRead = super.readDefaultLine(str, offsets, maxLineLength, maxBytesToConsume); - } - return bytesRead; - } - - @Override - public boolean needAdditionalRecordAfterSplit() { - return !finished && needAdditionalRecord; - } -} diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/LineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/LineReader.java deleted file mode 100644 index 0f31bafb27..0000000000 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/LineReader.java +++ /dev/null @@ -1,559 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; - -/** - * A class that provides a line reader from an input stream. - * Depending on the constructor used, lines will either be terminated by: - *
    - *
  • one of the following: '\n' (LF) , '\r' (CR), - * or '\r\n' (CR+LF).
    • - *
  • or, a custom byte sequence delimiter
    • - *
- * In both cases, EOF also terminates an otherwise unterminated - * line. - */ - -public class LineReader implements Closeable { - private static final int DEFAULT_BUFFER_SIZE = 64 * 1024; - private int bufferSize = DEFAULT_BUFFER_SIZE; - private InputStream in; - private byte[] buffer; - // the number of bytes of real data in the buffer - private int bufferLength = 0; - // the current position in the buffer - private int bufferPosn = 0; - - private static final byte CR = '\r'; - private static final byte LF = '\n'; - - // The line delimiter - private final byte[] recordDelimiterBytes; - - /** - * Create a line reader that reads from the given stream using the - * default buffer-size (64k). - * - * @param in The input stream - * @throws java.io.IOException - */ - public LineReader(InputStream in) { - this(in, DEFAULT_BUFFER_SIZE); - } - - /** - * Create a line reader that reads from the given stream using the - * given buffer-size. - * - * @param in The input stream - * @param bufferSize Size of the read buffer - * @throws java.io.IOException - */ - public LineReader(InputStream in, int bufferSize) { - this.in = in; - this.bufferSize = bufferSize; - this.buffer = new byte[this.bufferSize]; - this.recordDelimiterBytes = null; - } - - /** - * Create a line reader that reads from the given stream using the - * io.file.buffer.size specified in the given - * Configuration. - * - * @param in input stream - * @param conf configuration - * @throws java.io.IOException - */ - public LineReader(InputStream in, Configuration conf) throws IOException { - this(in, conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE)); - } - - /** - * Create a line reader that reads from the given stream using the - * default buffer-size, and using a custom delimiter of array of - * bytes. - * - * @param in The input stream - * @param recordDelimiterBytes The delimiter - */ - public LineReader(InputStream in, byte[] recordDelimiterBytes) { - this.in = in; - this.bufferSize = DEFAULT_BUFFER_SIZE; - this.buffer = new byte[this.bufferSize]; - this.recordDelimiterBytes = recordDelimiterBytes; - } - - /** - * Create a line reader that reads from the given stream using the - * given buffer-size, and using a custom delimiter of array of - * bytes. - * - * @param in The input stream - * @param bufferSize Size of the read buffer - * @param recordDelimiterBytes The delimiter - * @throws java.io.IOException - */ - public LineReader(InputStream in, int bufferSize, - byte[] recordDelimiterBytes) { - this.in = in; - this.bufferSize = bufferSize; - this.buffer = new byte[this.bufferSize]; - this.recordDelimiterBytes = recordDelimiterBytes; - } - - /** - * Create a line reader that reads from the given stream using the - * io.file.buffer.size specified in the given - * Configuration, and using a custom delimiter of array of - * bytes. - * - * @param in input stream - * @param conf configuration - * @param recordDelimiterBytes The delimiter - * @throws java.io.IOException - */ - public LineReader(InputStream in, Configuration conf, - byte[] recordDelimiterBytes) throws IOException { - this.in = in; - this.bufferSize = conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE); - this.buffer = new byte[this.bufferSize]; - this.recordDelimiterBytes = recordDelimiterBytes; - } - - - /** - * Close the underlying stream. - * - * @throws java.io.IOException - */ - public void close() throws IOException { - in.close(); - } - - public void reset() { - bufferLength = 0; - bufferPosn = 0; - - } - - /** - * Read one line from the InputStream into the given Text. - * - * @param str the object to store the given line (without newline) - * @param maxLineLength the maximum number of bytes to store into str; - * the rest of the line is silently discarded. - * @param maxBytesToConsume the maximum number of bytes to consume - * in this call. This is only a hint, because if the line cross - * this threshold, we allow it to happen. It can overshoot - * potentially by as much as one buffer length. - * @return the number of bytes read including the (longest) newline - * found. - * @throws java.io.IOException if the underlying stream throws - */ - public int readLine(Text str, int maxLineLength, - int maxBytesToConsume) throws IOException { - if (this.recordDelimiterBytes != null) { - return readCustomLine(str, maxLineLength, maxBytesToConsume); - } else { - return readDefaultLine(str, maxLineLength, maxBytesToConsume); - } - } - - protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter) - throws IOException { - return in.read(buffer); - } - /** - * Read a line terminated by one of CR, LF, or CRLF. - */ - private int readDefaultLine(Text str, int maxLineLength, int maxBytesToConsume) - throws IOException { - /* We're reading data from in, but the head of the stream may be - * already buffered in buffer, so we have several cases: - * 1. No newline characters are in the buffer, so we need to copy - * everything and read another buffer from the stream. - * 2. An unambiguously terminated line is in buffer, so we just - * copy to str. - * 3. Ambiguously terminated line is in buffer, i.e. buffer ends - * in CR. In this case we copy everything up to CR to str, but - * we also need to see what follows CR: if it's LF, then we - * need consume LF as well, so next call to readLine will read - * from after that. - * We use a flag prevCharCR to signal if previous character was CR - * and, if it happens to be at the end of the buffer, delay - * consuming it until we have a chance to look at the char that - * follows. - */ - str.clear(); - int txtLength = 0; //tracks str.getLength(), as an optimization - int newlineLength = 0; //length of terminating newline - boolean prevCharCR = false; //true of prev char was CR - long bytesConsumed = 0; - do { - int startPosn = bufferPosn; //starting from where we left off the last time - if (bufferPosn >= bufferLength) { - startPosn = bufferPosn = 0; - if (prevCharCR) { - ++bytesConsumed; //account for CR from previous read - } - bufferLength = fillBuffer(in, buffer, prevCharCR); - if (bufferLength <= 0) { - break; // EOF - } - } - for (; bufferPosn < bufferLength; ++bufferPosn) { //search for newline - if (buffer[bufferPosn] == LF) { - newlineLength = (prevCharCR) ? 2 : 1; - ++bufferPosn; // at next invocation proceed from following byte - break; - } - if (prevCharCR) { //CR + notLF, we are at notLF - newlineLength = 1; - break; - } - prevCharCR = (buffer[bufferPosn] == CR); - } - int readLength = bufferPosn - startPosn; - if (prevCharCR && newlineLength == 0) { - --readLength; //CR at the end of the buffer - } - bytesConsumed += readLength; - int appendLength = readLength - newlineLength; - if (appendLength > maxLineLength - txtLength) { - appendLength = maxLineLength - txtLength; - } - if (appendLength > 0) { - str.append(buffer, startPosn, appendLength); - txtLength += appendLength; - } - } while (newlineLength == 0 && bytesConsumed < maxBytesToConsume); - - if (bytesConsumed > (long) Integer.MAX_VALUE) { - throw new IOException("Too many bytes before newline: " + bytesConsumed); - } - return (int) bytesConsumed; - } - - /** - * Read a line terminated by one of CR, LF, or CRLF. - */ - public int readDefaultLine(NonSyncByteArrayOutputStream str, ArrayList offsets, int maxLineLength - , int maxBytesToConsume) - throws IOException { - /* We're reading data from in, but the head of the stream may be - * already buffered in buffer, so we have several cases: - * 1. No newline characters are in the buffer, so we need to copy - * everything and read another buffer from the stream. - * 2. An unambiguously terminated line is in buffer, so we just - * copy to str. - * 3. Ambiguously terminated line is in buffer, i.e. buffer ends - * in CR. In this case we copy everything up to CR to str, but - * we also need to see what follows CR: if it's LF, then we - * need consume LF as well, so next call to readLine will read - * from after that. - * We use a flag prevCharCR to signal if previous character was CR - * and, if it happens to be at the end of the buffer, delay - * consuming it until we have a chance to look at the char that - * follows. - */ - - int txtLength = 0; //tracks str.getLength(), as an optimization - int newlineLength = 0; //length of terminating newline - boolean prevCharCR = false; //true of prev char was CR - long bytesConsumed = 0; - do { - int startPosn = bufferPosn; //starting from where we left off the last time - if (bufferPosn >= bufferLength) { - startPosn = bufferPosn = 0; - if (prevCharCR) { - ++bytesConsumed; //account for CR from previous read - } - bufferLength = fillBuffer(in, buffer, prevCharCR); - if (bufferLength <= 0) { - break; // EOF - } - } - for (; bufferPosn < bufferLength; ++bufferPosn) { //search for newline - if (buffer[bufferPosn] == LF) { - newlineLength = (prevCharCR) ? 2 : 1; - ++bufferPosn; // at next invocation proceed from following byte - break; - } - if (prevCharCR) { //CR + notLF, we are at notLF - newlineLength = 1; - break; - } - prevCharCR = (buffer[bufferPosn] == CR); - } - int readLength = bufferPosn - startPosn; - if (prevCharCR && newlineLength == 0) { - --readLength; //CR at the end of the buffer - } - bytesConsumed += readLength; - int appendLength = readLength - newlineLength; - if (appendLength > maxLineLength - txtLength) { - appendLength = maxLineLength - txtLength; - } - if (appendLength > 0) { - str.write(buffer, startPosn, appendLength); - txtLength += appendLength; - } - } while (newlineLength == 0 && bytesConsumed < maxBytesToConsume); - - if (bytesConsumed > (long) Integer.MAX_VALUE) { - throw new IOException("Too many bytes before newline: " + bytesConsumed); - } - - if (bytesConsumed > 0) offsets.add(txtLength); - return (int) bytesConsumed; - } - - /** - * Read a line terminated by one of CR, LF, or CRLF. - */ - -/* int validIdx = 0; - public int readDefaultLines(NonSyncByteArrayOutputStream str, ArrayList offsets, ArrayList foffsets, - long pos, int maxLineLength, int maxBytesToConsume) - throws IOException { - *//* We're reading data from in, but the head of the stream may be - * already buffered in buffer, so we have several cases: - * 1. No newline characters are in the buffer, so we need to copy - * everything and read another buffer from the stream. - * 2. An unambiguously terminated line is in buffer, so we just - * copy to str. - * 3. Ambiguously terminated line is in buffer, i.e. buffer ends - * in CR. In this case we copy everything up to CR to str, but - * we also need to see what follows CR: if it's LF, then we - * need consume LF as well, so next call to readLine will read - * from after that. - * We use a flag prevCharCR to signal if previous character was CR - * and, if it happens to be at the end of the buffer, delay - * consuming it until we have a chance to look at the char that - * follows. - *//* - //str.clear(); - str.reset(); - offsets.clear(); - foffsets.clear(); - - validIdx = 0; - long bufferBytesConsumed = 0; - - int txtLength = 0; //tracks str.getLength(), as an optimization - int newlineLength = 0; //length of terminating newline - boolean prevCharCR = false; //true of prev char was CR - long bytesConsumed = 0; - do { - - int startPosn = bufferPosn; //starting from where we left off the last time - if (bufferPosn >= bufferLength) { - startPosn = bufferPosn = 0; - if (prevCharCR) { - ++bytesConsumed; //account for CR from previous read - } - bufferLength = in.read(buffer); - if (bufferLength <= 0) { - break; // EOF - } - } - for (; bufferPosn < bufferLength; ++bufferPosn) { //search for newline - if (buffer[bufferPosn] == LF) { - newlineLength = (prevCharCR) ? 2 : 1; - ++bufferPosn; // at next invocation proceed from following byte - break; - } - if (prevCharCR) { //CR + notLF, we are at notLF - newlineLength = 1; - break; - } - prevCharCR = (buffer[bufferPosn] == CR); - } - int readLength = bufferPosn - startPosn; - if (prevCharCR && newlineLength == 0) { - --readLength; //CR at the end of the buffer - } - bytesConsumed += readLength; - int appendLength = readLength - newlineLength; - if (appendLength > maxLineLength - txtLength) { - appendLength = maxLineLength - txtLength; - } - - if (appendLength > 0) { - str.write(buffer, startPosn, appendLength); - //System.out.println(startPosn + "," + appendLength); - //str.append(buffer, startPosn, appendLength); - txtLength += appendLength; - } - - if(newlineLength > 0){ - validIdx++; - - if (bytesConsumed > (long)Integer.MAX_VALUE) { - throw new IOException("Too many bytes before newline: " + bytesConsumed); - } - offsets.add(txtLength); - foffsets.add(pos); - pos+= bytesConsumed; - bufferBytesConsumed += bytesConsumed; - - txtLength = 0; - newlineLength = 0; - prevCharCR = false; //true of prev char was CR - bytesConsumed = 0; - } else { - bufferBytesConsumed += bytesConsumed; - bytesConsumed = 0; - } - } while ((bufferBytesConsumed < 256 * 1024)); - - return (int)bufferBytesConsumed; - }*/ - - /** - * Read a line terminated by a custom delimiter. - */ - private int readCustomLine(Text str, int maxLineLength, int maxBytesToConsume) - throws IOException { - /* We're reading data from inputStream, but the head of the stream may be - * already captured in the previous buffer, so we have several cases: - * - * 1. The buffer tail does not contain any character sequence which - * matches with the head of delimiter. We count it as a - * ambiguous byte count = 0 - * - * 2. The buffer tail contains a X number of characters, - * that forms a sequence, which matches with the - * head of delimiter. We count ambiguous byte count = X - * - * // *** eg: A segment of input file is as follows - * - * " record 1792: I found this bug very interesting and - * I have completely read about it. record 1793: This bug - * can be solved easily record 1794: This ." - * - * delimiter = "record"; - * - * supposing:- String at the end of buffer = - * "I found this bug very interesting and I have completely re" - * There for next buffer = "ad about it. record 179 ...." - * - * The matching characters in the input - * buffer tail and delimiter head = "re" - * Therefore, ambiguous byte count = 2 **** // - * - * 2.1 If the following bytes are the remaining characters of - * the delimiter, then we have to capture only up to the starting - * position of delimiter. That means, we need not include the - * ambiguous characters in str. - * - * 2.2 If the following bytes are not the remaining characters of - * the delimiter ( as mentioned in the example ), - * then we have to include the ambiguous characters in str. - */ - str.clear(); - int txtLength = 0; // tracks str.getLength(), as an optimization - long bytesConsumed = 0; - int delPosn = 0; - int ambiguousByteCount = 0; // To capture the ambiguous characters count - do { - int startPosn = bufferPosn; // Start from previous end position - if (bufferPosn >= bufferLength) { - startPosn = bufferPosn = 0; - bufferLength = fillBuffer(in, buffer, ambiguousByteCount > 0); - if (bufferLength <= 0) { - str.append(recordDelimiterBytes, 0, ambiguousByteCount); - break; // EOF - } - } - for (; bufferPosn < bufferLength; ++bufferPosn) { - if (buffer[bufferPosn] == recordDelimiterBytes[delPosn]) { - delPosn++; - if (delPosn >= recordDelimiterBytes.length) { - bufferPosn++; - break; - } - } else if (delPosn != 0) { - bufferPosn--; - delPosn = 0; - } - } - int readLength = bufferPosn - startPosn; - bytesConsumed += readLength; - int appendLength = readLength - delPosn; - if (appendLength > maxLineLength - txtLength) { - appendLength = maxLineLength - txtLength; - } - if (appendLength > 0) { - if (ambiguousByteCount > 0) { - str.append(recordDelimiterBytes, 0, ambiguousByteCount); - //appending the ambiguous characters (refer case 2.2) - bytesConsumed += ambiguousByteCount; - ambiguousByteCount = 0; - } - str.append(buffer, startPosn, appendLength); - txtLength += appendLength; - } - if (bufferPosn >= bufferLength) { - if (delPosn > 0 && delPosn < recordDelimiterBytes.length) { - ambiguousByteCount = delPosn; - bytesConsumed -= ambiguousByteCount; //to be consumed in next - } - } - } while (delPosn < recordDelimiterBytes.length - && bytesConsumed < maxBytesToConsume); - if (bytesConsumed > (long) Integer.MAX_VALUE) { - throw new IOException("Too many bytes before delimiter: " + bytesConsumed); - } - return (int) bytesConsumed; - } - - /** - * Read from the InputStream into the given Text. - * - * @param str the object to store the given line - * @param maxLineLength the maximum number of bytes to store into str. - * @return the number of bytes read including the newline - * @throws java.io.IOException if the underlying stream throws - */ - public int readLine(Text str, int maxLineLength) throws IOException { - return readLine(str, maxLineLength, Integer.MAX_VALUE); - } - - /** - * Read from the InputStream into the given Text. - * - * @param str the object to store the given line - * @return the number of bytes read including the newline - * @throws java.io.IOException if the underlying stream throws - */ - public int readLine(Text str) throws IOException { - return readLine(str, Integer.MAX_VALUE, Integer.MAX_VALUE); - } -} diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/SplitLineReader.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/SplitLineReader.java deleted file mode 100644 index 3579674ecf..0000000000 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/SplitLineReader.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.hadoop.conf.Configuration; - -import java.io.IOException; -import java.io.InputStream; - -public class SplitLineReader extends LineReader { - public SplitLineReader(InputStream in, byte[] recordDelimiterBytes) { - super(in, recordDelimiterBytes); - } - - public SplitLineReader(InputStream in, Configuration conf, - byte[] recordDelimiterBytes) throws IOException { - super(in, conf, recordDelimiterBytes); - } - - public boolean needAdditionalRecordAfterSplit() { - return false; - } -} diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index f50a20d46f..9740b4dea4 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -128,9 +128,7 @@ private void storageCompressionTest(String storeType, Class generateParameters() { return Arrays.asList(new Object[][] { - {"CSV"}, + {"TEXT"}, {"RAW"}, {"RCFILE"}, {"PARQUET"}, @@ -201,6 +201,7 @@ public void testMultipleFiles() throws IOException { private static boolean isProjectableStorage(String type) { if (type.equalsIgnoreCase("RCFILE") || type.equalsIgnoreCase("PARQUET") || + type.equalsIgnoreCase("TEXT") || type.equalsIgnoreCase("AVRO")) { return true; } else { diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index 72810fd8ec..fac7d7751b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -37,7 +37,6 @@ import java.io.IOException; -import static org.apache.tajo.storage.CSVFile.CSVScanner; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -111,7 +110,7 @@ public void testFindValueInSingleCSV() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner fileScanner = new CSVScanner(conf, schema, meta, tablet); + SeekableScanner fileScanner = OldStorageManager.getSeekableScanner(conf, meta, schema, tablet, schema); fileScanner.init(); Tuple keyTuple; long offset; @@ -135,7 +134,7 @@ public void testFindValueInSingleCSV() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindValueInCSV.idx"), keySchema, comp); reader.open(); - fileScanner = new CSVScanner(conf, schema, meta, tablet); + fileScanner = OldStorageManager.getSeekableScanner(conf, meta, schema, tablet, schema); fileScanner.init(); for (int i = 0; i < TUPLE_NUM - 1; i++) { tuple.put(0, DatumFactory.createInt8(i)); @@ -200,7 +199,7 @@ public void testFindNextKeyValueInSingleCSV() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner fileScanner = new CSVScanner(conf, schema, meta, tablet); + SeekableScanner fileScanner = OldStorageManager.getSeekableScanner(conf, meta, schema, tablet, schema); fileScanner.init(); Tuple keyTuple; long offset; @@ -221,7 +220,7 @@ public void testFindNextKeyValueInSingleCSV() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindNextKeyValueInCSV.idx"), keySchema, comp); reader.open(); - fileScanner = new CSVScanner(conf, schema, meta, tablet); + fileScanner = OldStorageManager.getSeekableScanner(conf, meta, schema, tablet, schema); fileScanner.init(); Tuple result; for(int i = 0 ; i < TUPLE_NUM -1 ; i ++) { diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml index 6a9e7ce75e..2de1617ee3 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml +++ b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml @@ -38,7 +38,7 @@ tajo.storage.scanner-handler - text,csv,json,raw,rcfile,row,parquet,sequencefile,avro + text,json,raw,rcfile,row,parquet,sequencefile,avro @@ -46,10 +46,6 @@ tajo.storage.fragment.text.class org.apache.tajo.storage.fragment.FileFragment - - tajo.storage.fragment.csv.class - org.apache.tajo.storage.fragment.FileFragment - tajo.storage.fragment.json.class org.apache.tajo.storage.fragment.FileFragment @@ -85,11 +81,6 @@ org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner - - tajo.storage.scanner-handler.csv.class - org.apache.tajo.storage.CSVFile$CSVScanner - - tajo.storage.scanner-handler.json.class org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner @@ -128,7 +119,7 @@ tajo.storage.appender-handler - text,csv,raw,rcfile,row,parquet,sequencefile,avro + text,raw,rcfile,row,parquet,sequencefile,avro @@ -136,11 +127,6 @@ org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender - - tajo.storage.appender-handler.csv.class - org.apache.tajo.storage.CSVFile$CSVAppender - - tajo.storage.appender-handler.json.class org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender From 8d5ef7a52e62938afafbbc9636355ce81173bedc Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 21 Jul 2015 17:31:34 +0900 Subject: [PATCH 2/3] TAJO-1496: Remove legacy CSVFile --- .../org/apache/tajo/catalog/CatalogUtil.java | 16 +++++---- .../org/apache/tajo/catalog/DDLBuilder.java | 2 +- .../src/main/proto/CatalogProtos.proto | 9 +++-- .../apache/tajo/catalog/TestTableDesc.java | 4 +-- .../apache/tajo/catalog/TestTableMeta.java | 18 +++++----- .../tajo/catalog/store/HiveCatalogStore.java | 3 +- .../tajo/catalog/store/HiveCatalogUtil.java | 4 +-- .../catalog/store/TestHiveCatalogStore.java | 12 +++---- .../org/apache/tajo/catalog/TestCatalog.java | 24 ++++++------- .../java/org/apache/tajo/BuiltinStorages.java | 1 - .../apache/tajo/storage/StorageConstants.java | 6 ---- .../java/org/apache/tajo/benchmark/TPCH.java | 2 +- .../tajo/engine/parser/SQLAnalyzer.java | 6 +--- .../engine/planner/global/GlobalPlanner.java | 6 ++-- .../planner/physical/PhysicalPlanUtil.java | 4 +-- .../tajo/master/exec/QueryExecutor.java | 7 ++-- .../org/apache/tajo/querymaster/Stage.java | 2 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../org/apache/tajo/QueryTestCaseBase.java | 2 +- .../org/apache/tajo/TajoTestingCluster.java | 2 +- .../apache/tajo/cli/tools/TestDDLBuilder.java | 2 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 12 +++---- .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 2 +- .../engine/planner/TestLogicalOptimizer.java | 6 ++-- .../engine/planner/TestLogicalPlanner.java | 12 +++---- .../tajo/engine/planner/TestPlannerUtil.java | 6 ++-- .../planner/physical/TestBNLJoinExec.java | 4 +-- .../planner/physical/TestBSTIndexExec.java | 2 +- .../physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 8 ++--- .../physical/TestFullOuterMergeJoinExec.java | 10 +++--- .../physical/TestHashAntiJoinExec.java | 4 +-- .../planner/physical/TestHashJoinExec.java | 4 +-- .../physical/TestHashSemiJoinExec.java | 4 +-- .../physical/TestLeftOuterHashJoinExec.java | 8 ++--- .../planner/physical/TestMergeJoinExec.java | 4 +-- .../planner/physical/TestNLJoinExec.java | 4 +-- .../planner/physical/TestPhysicalPlanner.java | 10 +++--- .../physical/TestRightOuterHashJoinExec.java | 6 ++-- .../physical/TestRightOuterMergeJoinExec.java | 10 +++--- .../engine/planner/physical/TestSortExec.java | 2 +- .../tajo/engine/query/TestCTASQuery.java | 2 +- .../tajo/engine/query/TestCreateTable.java | 4 +-- .../tajo/engine/query/TestHBaseTable.java | 36 +++++++++---------- .../tajo/engine/query/TestInsertQuery.java | 2 +- .../query/TestJoinOnPartitionedTables.java | 9 +++-- .../engine/query/TestTablePartitions.java | 32 ++++++++--------- .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../queries/TestAlterTable/table1_ddl.sql | 2 +- .../queries/TestAlterTable/table2_ddl.sql | 2 +- .../queries/TestCTASQuery/CtasWithOptions.sql | 2 +- .../quoted_identifier_mixed_chars_ddl_1.sql | 2 +- .../quoted_identifier_mixed_chars_ddl_2.sql | 2 +- .../quoted_identifier_non_ascii_ddl.sql | 2 +- .../queries/TestCreateTable/table1_ddl.sql | 2 +- ...InsertOverwriteLocationWithCompression.sql | 2 +- ...testInsertOverwriteWithCompression_ddl.sql | 4 +-- .../create_customer_large_ddl.sql | 2 +- .../create_lineitem_large_ddl.sql | 2 +- .../TestJoinQuery/create_orders_large_ddl.sql | 2 +- .../queries/TestJoinQuery/oj_table1_ddl.sql | 4 +-- .../queries/TestJoinQuery/oj_table2_ddl.sql | 4 +-- .../queries/TestJoinQuery/table1_int4_ddl.sql | 4 +-- .../queries/TestJoinQuery/table1_int8_ddl.sql | 4 +-- .../queries/TestNetTypes/table1_ddl.sql | 4 +-- .../queries/TestNetTypes/table2_ddl.sql | 4 +-- .../multibytes_delimiter_table1_ddl.sql | 4 +-- .../multibytes_delimiter_table2_ddl.sql | 4 +-- .../create_table_with_asc_desc_keys.sql | 2 +- .../create_table_with_date_ddl.sql | 4 +-- .../lineitemspecial_ddl.sql | 2 +- .../create_table_with_date_ddl.sql | 4 +-- .../queries/TestTruncateTable/table2_ddl.sql | 2 +- .../queries/default/create_table_12.sql | 2 +- .../results/TestTajoCli/testDescTable.result | 4 +-- .../testDescTableForNestedSchema.result | 4 +-- .../results/TestTajoDump/testDump1.result | 4 +-- .../results/TestTajoDump/testDump2.result | 4 +-- .../testBuildDDLForBaseTable.result | 4 +-- .../testBuildDDLForExternalTable.result | 4 +-- .../testBuildDDLQuotedTableName1.result | 4 +-- .../testBuildDDLQuotedTableName2.result | 4 +-- .../sphinx/backup_and_restore/catalog.rst | 2 +- tajo-docs/src/main/sphinx/getting_started.rst | 2 +- .../src/main/sphinx/index/how_to_use.rst | 2 +- .../src/main/sphinx/tsql/meta_command.rst | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 10 +++--- .../apache/tajo/plan/TablePropertyUtil.java | 2 +- .../plan/logical/PersistentStoreNode.java | 2 +- .../org/apache/tajo/plan/TestLogicalNode.java | 6 ++-- .../tajo/storage/hbase/HBaseFragment.java | 7 ++-- .../tajo/storage/fragment/FileFragment.java | 7 ++-- .../tajo/storage/TestCompressionStorages.java | 13 ++----- .../apache/tajo/storage/TestFileSystems.java | 2 +- .../tajo/storage/TestFileTablespace.java | 6 ++-- .../org/apache/tajo/storage/TestStorages.java | 5 ++- .../tajo/storage/index/TestBSTIndex.java | 1 - .../index/TestSingleCSVFileBSTIndex.java | 5 +-- 100 files changed, 248 insertions(+), 277 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 60d9b6b74f..52285fb7f2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.partition.PartitionDesc; @@ -53,8 +54,6 @@ public class CatalogUtil { - public static final String TEXTFILE_NAME = "TEXT"; - /** * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. * @@ -282,16 +281,21 @@ public static String getCanonicalTableName(String databaseName, String tableName return sb.toString(); } + + public static String getBackwardCompitablityStoreType(String storeType) { + return getStoreTypeString(getStoreType(storeType)); + } + public static String getStoreTypeString(final StoreType type) { if (type == StoreType.TEXTFILE) { - return TEXTFILE_NAME; + return BuiltinStorages.TEXT; } else { return type.name(); } } public static StoreType getStoreType(final String typeStr) { - if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) { + if (typeStr.equalsIgnoreCase("CSV")) { return StoreType.TEXTFILE; } else if (typeStr.equalsIgnoreCase(StoreType.RAW.name())) { return StoreType.RAW; @@ -305,7 +309,7 @@ public static StoreType getStoreType(final String typeStr) { return StoreType.SEQUENCEFILE; } else if (typeStr.equalsIgnoreCase(StoreType.AVRO.name())) { return StoreType.AVRO; - } else if (typeStr.equalsIgnoreCase(TEXTFILE_NAME)) { + } else if (typeStr.equalsIgnoreCase(BuiltinStorages.TEXT)) { return StoreType.TEXTFILE; } else if (typeStr.equalsIgnoreCase(StoreType.JSON.name())) { return StoreType.JSON; @@ -941,7 +945,7 @@ public static Pair, String> getPartitionKeyNamePair(Stri */ public static KeyValueSet newDefaultProperty(String storeType) { KeyValueSet options = new KeyValueSet(); - if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("TEXT")) { + if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT)) { options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); } else if (storeType.equalsIgnoreCase("JSON")) { options.set(StorageConstants.TEXT_SERDE_CLASS, "org.apache.tajo.storage.json.JsonLineSerDe"); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index 62dd894824..f313aa5e6b 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -87,7 +87,7 @@ public static void buildSchema(StringBuilder sb, Schema schema) { } private static void buildUsingClause(StringBuilder sb, TableMeta meta) { - sb.append(" USING " + meta.getStoreType()); + sb.append(" USING " + CatalogUtil.getBackwardCompitablityStoreType(meta.getStoreType())); } private static void buildWithClause(StringBuilder sb, TableMeta meta) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index d0cf238487..cb8c4030c0 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -27,7 +27,7 @@ import "PrimitiveProtos.proto"; enum StoreType { MEM = 0; - CSV = 1; + TEXTFILE = 1; RAW = 2; RCFILE = 3; ROWFILE = 4; @@ -35,10 +35,9 @@ enum StoreType { PARQUET = 7; SEQUENCEFILE = 8; AVRO = 9; - TEXTFILE = 10; - JSON = 11; - HBASE = 12; - SYSTEM = 13; + JSON = 10; + HBASE = 11; + SYSTEM = 12; } enum OrderType { diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java index 41a083201a..b3d343d85a 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java @@ -44,7 +44,7 @@ public void setup() throws IOException { schema = new Schema(); schema.addColumn("name", Type.BLOB); schema.addColumn("addr", Type.TEXT); - info = CatalogUtil.newTableMeta("CSV"); + info = CatalogUtil.newTableMeta("TEXT"); path = new Path(CommonTestingUtil.getTestDir(), "table1"); desc = new TableDesc("table1", schema, info, path.toUri()); stats = new TableStats(); @@ -70,7 +70,7 @@ public void test() throws CloneNotSupportedException, IOException { Schema schema = new Schema(); schema.addColumn("name", Type.BLOB); schema.addColumn("addr", Type.TEXT); - TableMeta info = CatalogUtil.newTableMeta("CSV"); + TableMeta info = CatalogUtil.newTableMeta("TEXT"); testClone(info); Path path = new Path(CommonTestingUtil.getTestDir(), "tajo"); diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java index 56c2ad2e91..9087262f7a 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java @@ -18,8 +18,8 @@ package org.apache.tajo.catalog; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.json.CatalogGsonHelper; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableProto; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; @@ -33,7 +33,7 @@ public class TestTableMeta { @Before public void setUp() { - meta = CatalogUtil.newTableMeta("CSV"); + meta = CatalogUtil.newTableMeta("TEXT"); } @Test @@ -41,7 +41,7 @@ public void testTableMetaTableProto() { Schema schema1 = new Schema(); schema1.addColumn("name", Type.BLOB); schema1.addColumn("addr", Type.TEXT); - TableMeta meta1 = CatalogUtil.newTableMeta("CSV"); + TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); TableMeta meta2 = new TableMeta(meta1.getProto()); assertEquals(meta1, meta2); @@ -52,7 +52,7 @@ public final void testClone() throws CloneNotSupportedException { Schema schema1 = new Schema(); schema1.addColumn("name", Type.BLOB); schema1.addColumn("addr", Type.TEXT); - TableMeta meta1 = CatalogUtil.newTableMeta("CSV"); + TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); TableMeta meta2 = (TableMeta) meta1.clone(); assertEquals(meta1.getStoreType(), meta2.getStoreType()); @@ -64,7 +64,7 @@ public void testSchema() throws CloneNotSupportedException { Schema schema1 = new Schema(); schema1.addColumn("name", Type.BLOB); schema1.addColumn("addr", Type.TEXT); - TableMeta meta1 = CatalogUtil.newTableMeta("CSV"); + TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); TableMeta meta2 = (TableMeta) meta1.clone(); @@ -73,7 +73,7 @@ public void testSchema() throws CloneNotSupportedException { @Test public void testGetStorageType() { - assertEquals("CSV", meta.getStoreType()); + assertEquals("TEXT", meta.getStoreType()); } @Test @@ -81,7 +81,7 @@ public void testEqualsObject() { Schema schema2 = new Schema(); schema2.addColumn("name", Type.BLOB); schema2.addColumn("addr", Type.TEXT); - TableMeta meta2 = CatalogUtil.newTableMeta("CSV"); + TableMeta meta2 = CatalogUtil.newTableMeta("TEXT"); assertTrue(meta.equals(meta2)); @@ -96,7 +96,7 @@ public void testEqualsObject2() { int MAX_COUNT = 17; - TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV.toString()); + TableMeta meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); for (int i = 0; i < MAX_COUNT; i++) { meta1.putOption("key"+i, "value"+i); } @@ -108,7 +108,7 @@ public void testEqualsObject2() { optionBuilder.addKeyval(keyValueBuilder); } TableProto.Builder builder = TableProto.newBuilder(); - builder.setStoreType(StoreType.CSV.toString()); + builder.setStoreType(BuiltinStorages.TEXT); builder.setParams(optionBuilder); TableMeta meta2 = new TableMeta(builder.build()); assertTrue(meta1.equals(meta2)); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index daba8bacbf..d593e4c445 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -489,8 +489,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); } - } else if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.CSV) - || tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.TEXTFILE)) { + } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.TEXTFILE)) { sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName()); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index ef7d4bd1de..5ae5969cbf 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -25,9 +25,9 @@ import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.ExceptionUtil; import org.apache.thrift.TException; @@ -113,7 +113,7 @@ public static String getStoreType(String fileFormat) { String outputFormatClass = fileFormatArrary[fileFormatArrary.length-1]; if(outputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) { - return CatalogUtil.TEXTFILE_NAME; + return BuiltinStorages.TEXT; } else if(outputFormatClass.equals(HiveSequenceFileOutputFormat.class.getSimpleName())) { return CatalogProtos.StoreType.SEQUENCEFILE.name(); } else if(outputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 1a8282a83b..b3af179421 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -85,7 +85,7 @@ public static void tearDown() throws IOException { @Test public void testTableUsingTextFile() throws Exception { - TableMeta meta = new TableMeta("CSV", new KeyValueSet()); + TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); @@ -177,7 +177,7 @@ public void testTableWithNullValue() throws Exception { KeyValueSet options = new KeyValueSet(); options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava("\u0002")); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava("\u0003")); - TableMeta meta = new TableMeta("CSV", options); + TableMeta meta = new TableMeta("TEXT", options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4); @@ -220,7 +220,7 @@ public void testTableWithNullValue() throws Exception { @Test public void testAddTableByPartition() throws Exception { - TableMeta meta = new TableMeta("CSV", new KeyValueSet()); + TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("n_name", TajoDataTypes.Type.TEXT); @@ -332,7 +332,7 @@ private void testDropPartition(String tableName, String partitionName) throws E @Test public void testGetAllTableNames() throws Exception{ - TableMeta meta = new TableMeta("CSV", new KeyValueSet()); + TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("n_name", TajoDataTypes.Type.TEXT); schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); @@ -360,7 +360,7 @@ public void testGetAllTableNames() throws Exception{ @Test public void testDeleteTable() throws Exception { - TableMeta meta = new TableMeta("CSV", new KeyValueSet()); + TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("n_name", TajoDataTypes.Type.TEXT); schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); @@ -472,7 +472,7 @@ public void testTableUsingParquet() throws Exception { public void testDataTypeCompatibility() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testDataTypeCompatibility"); - TableMeta meta = new TableMeta("CSV", new KeyValueSet()); + TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("col1", TajoDataTypes.Type.INT4); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 6020e830ec..40b731e9d2 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -264,7 +264,7 @@ private TableDesc createMockupTable(String databaseName, String tableName) throw TableDesc table = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), schema1, - new TableMeta("CSV", new KeyValueSet()), + new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true); return table; } @@ -317,7 +317,7 @@ public void testCreateAndDropTableWithCharacterSensivity() throws Exception { TableDesc table = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), schema, - new TableMeta("CSV", new KeyValueSet()), + new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true); assertTrue(catalog.createTable(table)); @@ -331,7 +331,7 @@ public void testCreateAndDropTableWithCharacterSensivity() throws Exception { table = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), schema, - new TableMeta("CSV", new KeyValueSet()), + new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true); assertTrue(catalog.createTable(table)); @@ -421,7 +421,7 @@ public void testGetTable() throws Exception { TableDesc meta = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"), schema1, - "CSV", + "TEXT", new KeyValueSet(), path.toUri()); @@ -441,7 +441,7 @@ private static void assertSchemaEquality(String tableName, Schema schema) throws TableDesc tableDesc = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName), schema, - "CSV", + "TEXT", new KeyValueSet(), path.toUri()); @@ -561,7 +561,7 @@ public static TableDesc prepareTable() throws IOException { String tableName = "indexed"; - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); return new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta, new Path(CommonTestingUtil.getTestDir(), "indexed").toUri()); @@ -732,7 +732,7 @@ public final void testAddAndDeleteTablePartitionByHash1() throws Exception { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); + TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = new Schema(); @@ -772,7 +772,7 @@ public final void testAddAndDeleteTablePartitionByHash2() throws Exception { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); + TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -810,7 +810,7 @@ public final void testAddAndDeleteTablePartitionByList() throws Exception { String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); + TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -847,7 +847,7 @@ public final void testAddAndDeleteTablePartitionByRange() throws Exception { String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); + TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -884,7 +884,7 @@ public final void testAddAndDeleteTablePartitionByColumn() throws Exception { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); + TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -1004,7 +1004,7 @@ public void testAlterTableName () throws Exception { TableDesc setPropertyDesc = catalog.getTableDesc("default","mynewcooltable"); KeyValueSet options = new KeyValueSet(); options.set("timezone", "GMT+9"); // Seoul, Korea - setPropertyDesc.setMeta(new TableMeta("CSV", options)); + setPropertyDesc.setMeta(new TableMeta("TEXT", options)); String prevTimeZone = setPropertyDesc.getMeta().getOption("timezone"); String newTimeZone = "GMT-7"; // Silicon Valley, California catalog.alterTable(createMockAlterTableSetProperty(newTimeZone)); diff --git a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java index 318270c08d..6f1b7c6502 100644 --- a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java +++ b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java @@ -19,7 +19,6 @@ package org.apache.tajo; public class BuiltinStorages { - public static final String CSV = "CSV"; public static final String TEXT = "TEXT"; public static final String JSON = "JSON"; public static final String RAW = "RAW"; diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java index 16cf51d145..1cf33d44d7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java @@ -18,8 +18,6 @@ package org.apache.tajo.storage; -import org.apache.tajo.TajoConstants; - import java.net.URI; public class StorageConstants { @@ -42,10 +40,6 @@ public class StorageConstants { // Text file properties ------------------------------------------------- @Deprecated - public static final String CSVFILE_DELIMITER = "csvfile.delimiter"; - @Deprecated - public static final String CSVFILE_NULL = "csvfile.null"; - @Deprecated public static final String CSVFILE_SERDE = "csvfile.serde"; public static final String TEXT_DELIMITER = "text.delimiter"; diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java index dd3a43b813..91a3b66bbe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java +++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java @@ -207,7 +207,7 @@ public void loadTables() throws SQLException { } public void loadTable(String tableName) throws SQLException { - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); meta.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); PartitionMethodDesc partitionMethodDesc = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index e433e4f6ea..d99a5cf291 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1623,12 +1623,8 @@ private Map getParams(SQLParser.Param_clauseContext ctx) { public Map escapeTableMeta(Map map) { Map params = new HashMap(); for (Map.Entry entry : map.entrySet()) { - if (entry.getKey().equals(StorageConstants.CSVFILE_DELIMITER) - || entry.getKey().equals(StorageConstants.TEXT_DELIMITER)) { //backward compatibility + if (entry.getKey().equals(StorageConstants.TEXT_DELIMITER)) { params.put(StorageConstants.TEXT_DELIMITER, StringUtils.unicodeEscapedDelimiter(entry.getValue())); - } else if (entry.getKey().equals(StorageConstants.CSVFILE_NULL) - || entry.getKey().equals(StorageConstants.TEXT_NULL)) { //backward compatibility - params.put(StorageConstants.TEXT_NULL, entry.getValue()); } else { params.put(entry.getKey(), entry.getValue()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index ba4833b215..a6735d7f94 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -23,10 +23,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; @@ -162,7 +160,7 @@ public void build(QueryContext queryContext, MasterPlan masterPlan) throws IOExc private static void setFinalOutputChannel(DataChannel outputChannel, Schema outputSchema) { outputChannel.setShuffleType(NONE_SHUFFLE); outputChannel.setShuffleOutputNum(1); - outputChannel.setStoreType("CSV"); + outputChannel.setStoreType("TEXT"); outputChannel.setSchema(outputSchema); } @@ -997,7 +995,7 @@ public LogicalNode visitProjection(GlobalPlanContext context, LogicalPlan plan, for (DataChannel dataChannel : masterPlan.getIncomingChannels(execBlock.getId())) { // This data channel will be stored in staging directory, but RawFile, default file type, does not support // distributed file system. It needs to change the file format for distributed file system. - dataChannel.setStoreType("CSV"); + dataChannel.setStoreType("TEXT"); ExecutionBlock subBlock = masterPlan.getExecBlock(dataChannel.getSrcId()); ProjectionNode copy = PlannerUtil.clone(plan, node); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index deda4988e5..1e90d0450c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -209,7 +209,7 @@ public PhysicalExec visit(PhysicalExec exec, Stack stack, Class columns = new ArrayList(); CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder() .setTableName(nodeUniqName) - .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType("CSV").build()) + .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType(BuiltinStorages.TEXT).build()) .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build()) .setStats(stats.getProto()) .build(); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 50b45a8541..42565822d8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -742,7 +742,7 @@ private void finalizeStats() { String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); if (storeType == null) { // get default or store type - storeType = "CSV"; + storeType = "TEXT"; } schema = channel.getSchema(); diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index 5df112293a..050c7b40b4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -41,7 +41,7 @@ public class BackendTestingUtil { mockupSchema = new Schema(); mockupSchema.addColumn("deptname", Type.TEXT); mockupSchema.addColumn("score", Type.INT4); - mockupMeta = CatalogUtil.newTableMeta("CSV"); + mockupMeta = CatalogUtil.newTableMeta("TEXT"); } public static void writeTmpTable(TajoConf conf, Path tablePath) diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 295ddbc06c..86e0427e82 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -820,7 +820,7 @@ public List executeDDL(String ddlFileName, @Nullable String [] args) thr * CREATE EXTERNAL TABLE ${0} ( * t_timestamp TIMESTAMP, * t_date DATE - * ) USING CSV LOCATION ${table.path} + * ) USING TEXT LOCATION ${table.path} * * * @param ddlFileName A file name, containing a data definition statement. diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 60349dd003..acd69b6d4f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -675,7 +675,7 @@ public static void createTable(String tableName, Schema schema, out.close(); } } - TableMeta meta = CatalogUtil.newTableMeta("CSV", tableOption); + TableMeta meta = CatalogUtil.newTableMeta("TEXT", tableOption); client.createExternalTable(tableName, schema, tablePath.toUri(), meta); } finally { client.close(); diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java index aac5f907f1..06a54c4d63 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java @@ -41,7 +41,7 @@ public class TestDDLBuilder { schema1.addColumn("name", TajoDataTypes.Type.BLOB); schema1.addColumn("addr", TajoDataTypes.Type.TEXT); - meta1 = CatalogUtil.newTableMeta("CSV"); + meta1 = CatalogUtil.newTableMeta("TEXT"); meta1.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName()); diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index c96311ee56..17688968b1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -35,10 +35,7 @@ import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.FileUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.rules.TestName; import java.io.*; @@ -396,17 +393,16 @@ public void testTimeZoneTest1() throws Exception { } @Test + @Ignore //MetaData should be fixed public void testTimeZoneTest2() throws Exception { String tableName = "test1"; - tajoCli.executeMetaCommand("\\set TIMEZONE GMT+0"); + tajoCli.executeMetaCommand("\\set TIMEZONE GMT+1"); tajoCli.executeScript("create table " + tableName + " (col1 TIMESTAMP)"); tajoCli.executeScript("insert into " + tableName + " select to_timestamp(0)"); - tajoCli.executeMetaCommand("\\set TIMEZONE GMT+1"); tajoCli.executeScript("select * from " + tableName); String consoleResult = new String(out.toByteArray()); tajoCli.executeScript("DROP TABLE " + tableName + " PURGE"); - System.out.println(consoleResult); - assertTrue(consoleResult.contains("1970-01-01 00:00:00")); + assertTrue(consoleResult.contains("1970-01-01 01:00:00")); } @Test(timeout = 3000) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index b9bdbf5938..f92356c21c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -264,7 +264,7 @@ public void testEval(OverridableConf context, Schema schema, String tableName, S vtuple.put(i, lazyTuple.get(i)); } } - cat.createTable(new TableDesc(qualifiedTableName, inputSchema,"CSV", + cat.createTable(new TableDesc(qualifiedTableName, inputSchema,"TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri())); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index 5a8238cae3..9106cbaac4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -103,7 +103,7 @@ public static void setUp() throws Exception { schema.addColumn("score", TajoDataTypes.Type.INT4); schema.addColumn("age", TajoDataTypes.Type.INT4); - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc desc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta, CommonTestingUtil.getTestDir().toUri()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index afa3472a6f..e845c03c8b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -79,7 +79,7 @@ public static void setUp() throws Exception { schema3.addColumn("score", Type.INT4); schema3.addColumn("phone", Type.INT4); - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); @@ -87,13 +87,13 @@ public static void setUp() throws Exception { TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index d1b0c37e0d..928b1643ac 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -96,19 +96,19 @@ public static void setUp() throws Exception { schema3.addColumn("deptname", Type.TEXT); schema3.addColumn("score", Type.INT4); - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); @@ -125,7 +125,7 @@ public static void setUp() throws Exception { tpch.loadSchemas(); tpch.loadOutSchema(); for (String table : tpchTables) { - TableMeta m = CatalogUtil.newTableMeta("CSV"); + TableMeta m = CatalogUtil.newTableMeta("TEXT"); TableDesc d = CatalogUtil.newTableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); @@ -990,7 +990,7 @@ public final void testCreateTableDef() throws PlanningException { assertEquals(Type.INT8, def.getColumn(2).getDataType().getType()); assertEquals("score", def.getColumn(3).getSimpleName()); assertEquals(Type.FLOAT4, def.getColumn(3).getDataType().getType()); - assertTrue("CSV".equalsIgnoreCase(createTable.getStorageType())); + assertTrue("TEXT".equalsIgnoreCase(createTable.getStorageType())); assertEquals("file://tmp/data", createTable.getUri().toString()); assertTrue(createTable.hasOptions()); assertEquals("|", createTable.getOptions().get("csv.delimiter")); @@ -1265,7 +1265,7 @@ public final void testAddPartitionAndDropPartition() throws PlanningException { TableDesc desc = null; try { - desc = new TableDesc(qualifiedTableName, schema, "CSV", new KeyValueSet(), + desc = new TableDesc(qualifiedTableName, schema, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); } catch (Exception e) { throw new PlanningException(e.getMessage()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index fb35220335..1311275529 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -85,7 +85,7 @@ public static void setUp() throws Exception { schema3.addColumn("deptname", Type.TEXT); schema3.addColumn("score", CatalogUtil.newSimpleDataType(Type.INT4)); - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); @@ -93,13 +93,13 @@ public static void setUp() throws Exception { TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index d1983266f2..8a3192922f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -84,7 +84,7 @@ public void setUp() throws Exception { schema.addColumn("memid", Type.INT4); schema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, schema, employeePath); @@ -106,7 +106,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("fk_memid", Type.INT4); peopleSchema.addColumn("name", Type.TEXT); peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); Path peoplePath = new Path(testDir, "people.csv"); appender = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index e5778818a2..036ca7821e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -111,7 +111,7 @@ public void setup() throws Exception { writer.open(); long offset; - meta = CatalogUtil.newTableMeta("CSV"); + meta = CatalogUtil.newTableMeta("TEXT"); tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv"); fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 2578de635f..1a40949710 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -80,7 +80,7 @@ public void setUp() throws Exception { schema.addColumn("empid", Type.INT4); schema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, schema, employeePath); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 3c9177ebdc..7eb8ed9aa2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -102,7 +102,7 @@ public void setUp() throws Exception { dep3Schema.addColumn("loc_id", Type.INT4); - TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); @@ -131,7 +131,7 @@ public void setUp() throws Exception { job3Schema.addColumn("job_title", Type.TEXT); - TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); @@ -170,7 +170,7 @@ public void setUp() throws Exception { emp3Schema.addColumn("job_id", Type.INT4); - TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); @@ -222,7 +222,7 @@ public void setUp() throws Exception { phone3Schema.addColumn("phone_number", Type.TEXT); - TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 6b32db0c6c..124ab17a96 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -107,7 +107,7 @@ public void setUp() throws Exception { dep3Schema.addColumn("loc_id", Type.INT4); - TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); @@ -145,7 +145,7 @@ public void setUp() throws Exception { dep4Schema.addColumn("loc_id", Type.INT4); - TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT"); Path dep4Path = new Path(testDir, "dep4.csv"); Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); @@ -176,7 +176,7 @@ public void setUp() throws Exception { job3Schema.addColumn("job_title", Type.TEXT); - TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); @@ -215,7 +215,7 @@ public void setUp() throws Exception { emp3Schema.addColumn("job_id", Type.INT4); - TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); @@ -267,7 +267,7 @@ public void setUp() throws Exception { phone3Schema.addColumn("phone_number", Type.TEXT); - TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 1b64a8fb15..6705b21e9d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -81,7 +81,7 @@ public void setUp() throws Exception { employeeSchema.addColumn("memid", Type.INT4); employeeSchema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, employeeSchema, employeePath); @@ -107,7 +107,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("fk_memid", Type.INT4); peopleSchema.addColumn("name", Type.TEXT); peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); Path peoplePath = new Path(testDir, "people.csv"); appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(peopleMeta, peopleSchema, peoplePath); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index f7bc1dde21..0777ea39e1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -83,7 +83,7 @@ public void setUp() throws Exception { employeeSchema.addColumn("memid", Type.INT4); employeeSchema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, employeeSchema, employeePath); @@ -106,7 +106,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("fk_memid", Type.INT4); peopleSchema.addColumn("name", Type.TEXT); peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); Path peoplePath = new Path(testDir, "people.csv"); appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(peopleMeta, peopleSchema, peoplePath); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index afa273b0bb..37eb9c9422 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -82,7 +82,7 @@ public void setUp() throws Exception { employeeSchema.addColumn("memid", Type.INT4); employeeSchema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, employeeSchema, employeePath); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("fk_memid", Type.INT4); peopleSchema.addColumn("name", Type.TEXT); peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); Path peoplePath = new Path(testDir, "people.csv"); appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(peopleMeta, peopleSchema, peoplePath); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 87bcd20539..4df3cd91bb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -103,7 +103,7 @@ public void setUp() throws Exception { dep3Schema.addColumn("loc_id", Type.INT4); - TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); @@ -132,7 +132,7 @@ public void setUp() throws Exception { job3Schema.addColumn("job_title", Type.TEXT); - TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(job3Meta, job3Schema, job3Path); @@ -172,7 +172,7 @@ public void setUp() throws Exception { emp3Schema.addColumn("job_id", Type.INT4); - TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(emp3Meta, emp3Schema, emp3Path); @@ -225,7 +225,7 @@ public void setUp() throws Exception { phone3Schema.addColumn("phone_number", Type.TEXT); - TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index d4189bb505..44142e3764 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -83,7 +83,7 @@ public void setUp() throws Exception { employeeSchema.addColumn("memid", Type.INT4); employeeSchema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, employeeSchema, employeePath); @@ -112,7 +112,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("fk_memid", Type.INT4); peopleSchema.addColumn("name", Type.TEXT); peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); Path peoplePath = new Path(testDir, "people.csv"); appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(peopleMeta, peopleSchema, peoplePath); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 1b30ef80d8..8fb5dcf413 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -81,7 +81,7 @@ public void setUp() throws Exception { schema.addColumn("memid", Type.INT4); schema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(employeeMeta, schema, employeePath); @@ -105,7 +105,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("fk_memid", Type.INT4); peopleSchema.addColumn("name", Type.TEXT); peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); Path peoplePath = new Path(testDir, "people.csv"); appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(peopleMeta, peopleSchema, peoplePath); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index ca4b6b79d2..31cb88b7d9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -117,7 +117,7 @@ public static void setUp() throws Exception { scoreSchema.addColumn("score", Type.INT4); scoreSchema.addColumn("nullable", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); Path employeePath = new Path(testDir, "employee.csv"); @@ -138,7 +138,7 @@ public static void setUp() throws Exception { catalog.createTable(employee); Path scorePath = new Path(testDir, "score"); - TableMeta scoreMeta = CatalogUtil.newTableMeta("CSV", new KeyValueSet()); + TableMeta scoreMeta = CatalogUtil.newTableMeta("TEXT", new KeyValueSet()); appender = sm.getAppender(scoreMeta, scoreSchema, scorePath); appender.init(); score = new TableDesc( @@ -434,7 +434,7 @@ public final void testStorePlan() throws IOException, PlanningException { LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); - TableMeta outputMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta outputMeta = CatalogUtil.newTableMeta("TEXT"); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -503,7 +503,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann long totalNum = 0; for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { Scanner scanner = ((FileTablespace) TablespaceManager.getLocalFs()).getFileScanner( - CatalogUtil.newTableMeta("CSV"), + CatalogUtil.newTableMeta("TEXT"), rootNode.getOutSchema(), status.getPath()); @@ -741,7 +741,7 @@ public final void testPartitionedStorePlanWithMaxFileSize() throws IOException, long expectedFileNum = (long) Math.ceil(fileVolumSum / (float)StorageUnit.MB); assertEquals(expectedFileNum, fileStatuses.length); } - TableMeta outputMeta = CatalogUtil.newTableMeta("CSV"); + TableMeta outputMeta = CatalogUtil.newTableMeta("TEXT"); Scanner scanner = new MergeScanner(conf, rootNode.getOutSchema(), outputMeta, TUtil.newList(fragments)); scanner.init(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index f581db86de..9c053a000b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -98,7 +98,7 @@ public void setUp() throws Exception { dep3Schema.addColumn("loc_id", Type.INT4); - TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep3Meta, dep3Schema, dep3Path); @@ -128,7 +128,7 @@ public void setUp() throws Exception { job3Schema.addColumn("job_title", Type.TEXT); - TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(job3Meta, job3Schema, job3Path); @@ -168,7 +168,7 @@ public void setUp() throws Exception { emp3Schema.addColumn("job_id", Type.INT4); - TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(emp3Meta, emp3Schema, emp3Path); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 51f0e764a0..461b2bfeaf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { dep3Schema.addColumn("loc_id", Type.INT4); - TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep3Meta, dep3Schema, dep3Path); @@ -144,7 +144,7 @@ public void setUp() throws Exception { dep4Schema.addColumn("loc_id", Type.INT4); - TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT"); Path dep4Path = new Path(testDir, "dep4.csv"); Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep4Meta, dep4Schema, dep4Path); @@ -176,7 +176,7 @@ public void setUp() throws Exception { job3Schema.addColumn("job_title", Type.TEXT); - TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(job3Meta, job3Schema, job3Path); @@ -216,7 +216,7 @@ public void setUp() throws Exception { emp3Schema.addColumn("job_id", Type.INT4); - TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(emp3Meta, emp3Schema, emp3Path); @@ -269,7 +269,7 @@ public void setUp() throws Exception { phone3Schema.addColumn("phone_number", Type.TEXT); - TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); + TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 4690e7148b..035791c692 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -76,7 +76,7 @@ public static void setUp() throws Exception { schema.addColumn("empid", Type.INT4); schema.addColumn("deptname", Type.TEXT); - employeeMeta = CatalogUtil.newTableMeta("CSV"); + employeeMeta = CatalogUtil.newTableMeta("TEXT"); tablePath = StorageUtil.concatPath(workDir, "employee", "table1"); sm.getFileSystem().mkdirs(tablePath.getParent()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index 2758d7d1f9..f327c85296 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -248,7 +248,7 @@ public final void testCtasWithOptions() throws Exception { TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); - assertTrue("CSV".equalsIgnoreCase(desc.getMeta().getStoreType())); + assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getStoreType())); KeyValueSet options = desc.getMeta().getOptions(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index dfdf09abd4..08aabf4a2c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -486,7 +486,7 @@ public final void testCreateTableLike1() throws Exception { executeString("DROP TABLE table2"); // Table with non-default meta options - executeString("CREATE TABLE table1 (c1 int, c2 text) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); + executeString("CREATE TABLE table1 (c1 int, c2 text) USING text WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); executeString("CREATE TABLE table2 LIKE table1"); testMsg = "testCreateTableLike1: Table with non-default meta options test failed"; assertTrue(testMsg, isClonedTable("table1","table2")); @@ -552,7 +552,7 @@ public final void testCreateTableLike1() throws Exception { executeString("DROP TABLE table2"); // Table with non-default meta options - executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); + executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING text WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); executeString("CREATE TABLE table2 LIKE table1"); testMsg = "testCreateTableLike1: Table with non-default meta options test failed"; assertTrue(testMsg, isClonedTable("table1","table2")); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 569111c595..7adb2379b1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -735,8 +735,8 @@ public void testInsertIntoMultiRegion() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); @@ -791,8 +791,8 @@ public void testInsertIntoMultiRegion2() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); @@ -849,8 +849,8 @@ public void testInsertIntoMultiRegionWithSplitFile() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); @@ -907,8 +907,8 @@ public void testInsertIntoMultiRegionMultiRowFields() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id1", Type.TEXT); @@ -964,8 +964,8 @@ public void testInsertIntoBinaryMultiRegion() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.INT4); @@ -1020,8 +1020,8 @@ public void testInsertIntoColumnKeyValue() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("rk", Type.TEXT); @@ -1109,8 +1109,8 @@ public void testInsertIntoDifferentType() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.INT4); @@ -1181,8 +1181,8 @@ public void testInsertIntoRowField() throws Exception { public void testCTAS() throws Exception { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); @@ -1302,8 +1302,8 @@ public void testInsertIntoLocation() throws Exception { try { // create test table KeyValueSet tableOptions = new KeyValueSet(); - tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java index 7a671d8386..d0088a52cf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java @@ -689,7 +689,7 @@ public final void testInsertOverwriteTableWithNonFromQuery2() throws Exception { public final void testInsertOverwritePathWithNonFromQuery() throws Exception { ResultSet res = executeString("insert overwrite into location " + "'/tajo-data/testInsertOverwritePathWithNonFromQuery' " + - "USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + "USING csv WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "select 1::INT4, 2.1::FLOAT4, 'test'"); res.close(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java index f605ccfaba..fdaac1d62f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java @@ -18,7 +18,6 @@ package org.apache.tajo.engine.query; -import com.google.protobuf.ServiceException; import org.apache.tajo.IntegrationTest; import org.apache.tajo.NamedTest; import org.apache.tajo.TajoTestingCluster; @@ -225,8 +224,8 @@ public final void testSelfJoin() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("paritioned_nation"); ResultSet res = executeString( "create table " + tableName + " (n_name text," - + " n_comment text, n_regionkey int8) USING csv " - + "WITH ('csvfile.delimiter'='|')" + + " n_comment text, n_regionkey int8) USING text " + + "WITH ('text.delimiter'='|')" + "PARTITION BY column(n_nationkey int8)"); res.close(); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); @@ -264,7 +263,7 @@ public final void testSelfJoin2() throws Exception { See the following case. CREATE TABLE orders_partition (o_orderkey INT8, o_custkey INT8, o_totalprice FLOAT8, o_orderpriority TEXT, - o_clerk TEXT, o_shippriority INT4, o_comment TEXT) USING CSV WITH ('csvfile.delimiter'='|') + o_clerk TEXT, o_shippriority INT4, o_comment TEXT) USING TEXT WITH ('text.delimiter'='|') PARTITION BY COLUMN(o_orderdate TEXT, o_orderstatus TEXT); select a.o_orderstatus, count(*) as cnt @@ -283,7 +282,7 @@ and a.o_orderstatus in ('F') String tableName = CatalogUtil.normalizeIdentifier("partitioned_orders"); ResultSet res = executeString( "create table " + tableName + " (o_orderkey INT8, o_custkey INT8, o_totalprice FLOAT8, o_orderpriority TEXT,\n" + - "o_clerk TEXT, o_shippriority INT4, o_comment TEXT) USING CSV WITH ('csvfile.delimiter'='|')\n" + + "o_clerk TEXT, o_shippriority INT4, o_comment TEXT) USING TEXT WITH ('text.delimiter'='|')\n" + "PARTITION BY COLUMN(o_orderdate TEXT, o_orderstatus TEXT, o_orderkey_mod INT8)"); res.close(); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index 77bfca6213..265c726100 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -592,8 +592,8 @@ public final void testColumnPartitionedTableByOneColumnsWithCompression() throws if (nodeType == NodeType.INSERT) { res = executeString( - "create table " + tableName + " (col2 int4, col3 float8) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + "create table " + tableName + " (col2 int4, col3 float8) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "PARTITION BY column(col1 int4)"); res.close(); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); @@ -602,8 +602,8 @@ public final void testColumnPartitionedTableByOneColumnsWithCompression() throws "insert overwrite into " + tableName + " select l_partkey, l_quantity, l_orderkey from lineitem"); } else { res = executeString( - "create table " + tableName + " (col2 int4, col3 float8) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + "create table " + tableName + " (col2 int4, col3 float8) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "PARTITION BY column(col1 int4) as select l_partkey, l_quantity, l_orderkey from lineitem"); } res.close(); @@ -639,8 +639,8 @@ public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByTwoColumnsWithCompression"); if (nodeType == NodeType.INSERT) { - res = executeString("create table " + tableName + " (col3 float8, col4 text) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + res = executeString("create table " + tableName + " (col3 float8, col4 text) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "PARTITION by column(col1 int4, col2 int4)"); res.close(); @@ -650,8 +650,8 @@ public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws "insert overwrite into " + tableName + " select l_quantity, l_returnflag, l_orderkey, l_partkey from lineitem"); } else { - res = executeString("create table " + tableName + " (col3 float8, col4 text) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + res = executeString("create table " + tableName + " (col3 float8, col4 text) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "PARTITION by column(col1 int4, col2 int4) as select l_quantity, l_returnflag, l_orderkey, " + "l_partkey from lineitem"); } @@ -696,8 +696,8 @@ public final void testColumnPartitionedTableByThreeColumnsWithCompression() thro if (nodeType == NodeType.INSERT) { res = executeString( - "create table " + tableName + " (col4 text) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + "create table " + tableName + " (col4 text) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "partition by column(col1 int4, col2 int4, col3 float8)"); res.close(); @@ -707,8 +707,8 @@ public final void testColumnPartitionedTableByThreeColumnsWithCompression() thro "insert overwrite into " + tableName + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); } else { - res = executeString("create table " + tableName + " (col4 text) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + res = executeString("create table " + tableName + " (col4 text) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "partition by column(col1 int4, col2 int4, col3 float8) as select l_returnflag, l_orderkey, l_partkey, " + "l_quantity from lineitem"); } @@ -791,8 +791,8 @@ public final void testColumnPartitionedTableNoMatchedPartition() throws Exceptio if (nodeType == NodeType.INSERT) { res = executeString( - "create table " + tableName + " (col4 text) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + "create table " + tableName + " (col4 text) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "partition by column(col1 int4, col2 int4, col3 float8)"); res.close(); @@ -802,8 +802,8 @@ public final void testColumnPartitionedTableNoMatchedPartition() throws Exceptio "insert overwrite into " + tableName + " select l_returnflag , l_orderkey, l_partkey, l_quantity from lineitem"); } else { - res = executeString("create table " + tableName + " (col4 text) USING csv " + - "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + + res = executeString("create table " + tableName + " (col4 text) USING text " + + "WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " + "partition by column(col1 int4, col2 int4, col3 float8) as select l_returnflag , l_orderkey, l_partkey, " + "l_quantity from lineitem"); } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 3d32c08828..0c83fd0575 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -71,7 +71,7 @@ public static void setup() throws Exception { scoreSchema = new Schema(); scoreSchema.addColumn("deptname", Type.TEXT); scoreSchema.addColumn("score", Type.INT4); - scoreMeta = CatalogUtil.newTableMeta("CSV"); + scoreMeta = CatalogUtil.newTableMeta("TEXT"); TableStats stats = new TableStats(); Path p = new Path(sm.getTableUri("default", "score")); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 7c61cc7177..169f375bff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -69,7 +69,7 @@ public static void setUp() throws Exception { tpch.loadSchemas(); tpch.loadOutSchema(); for (String table : tpch.getTableNames()) { - TableMeta m = CatalogUtil.newTableMeta("CSV"); + TableMeta m = CatalogUtil.newTableMeta("TEXT"); TableDesc d = CatalogUtil.newTableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); TableStats stats = new TableStats(); diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/table1_ddl.sql b/tajo-core/src/test/resources/queries/TestAlterTable/table1_ddl.sql index 8d7fba023b..f00f5933d1 100644 --- a/tajo-core/src/test/resources/queries/TestAlterTable/table1_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestAlterTable/table1_ddl.sql @@ -1 +1 @@ -CREATE EXTERNAL TABLE ${0} (id int, str text, num int) using csv location ${table.path}; +CREATE EXTERNAL TABLE ${0} (id int, str text, num int) using text location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql b/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql index ae563e751c..887029d93f 100644 --- a/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql @@ -1 +1 @@ -CREATE EXTERNAL TABLE ${0} (xx text, yy text, zz text) USING CSV WITH('text.delimiter'='+') LOCATION ${table.path}; +CREATE EXTERNAL TABLE ${0} (xx text, yy text, zz text) USING TEXT WITH('text.delimiter'='+') LOCATION ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql b/tajo-core/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql index a393b9d121..b7dde1586b 100644 --- a/tajo-core/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql +++ b/tajo-core/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql @@ -1,5 +1,5 @@ create table testCtasWithOptions (col1 float, col2 float) -using csv with ('csvfile.delimiter'='\u0001') as +using text with ('text.delimiter'='\u0001') as select sum(l_orderkey) as total1, avg(l_partkey) as total2 diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_1.sql b/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_1.sql index f93e1e6d5a..d8293d0fa0 100644 --- a/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_1.sql +++ b/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_1.sql @@ -1 +1 @@ -CREATE EXTERNAL TABLE ${0} ("aGe" int, "teXt" text, "Number" int) using csv location ${table.path}; \ No newline at end of file +CREATE EXTERNAL TABLE ${0} ("aGe" int, "teXt" text, "Number" int) using text location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_2.sql b/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_2.sql index f93e1e6d5a..d8293d0fa0 100644 --- a/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_2.sql +++ b/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_mixed_chars_ddl_2.sql @@ -1 +1 @@ -CREATE EXTERNAL TABLE ${0} ("aGe" int, "teXt" text, "Number" int) using csv location ${table.path}; \ No newline at end of file +CREATE EXTERNAL TABLE ${0} ("aGe" int, "teXt" text, "Number" int) using text location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_non_ascii_ddl.sql b/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_non_ascii_ddl.sql index c1f073da0c..1706d808ba 100644 --- a/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_non_ascii_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestCreateTable/quoted_identifier_non_ascii_ddl.sql @@ -1 +1 @@ -CREATE EXTERNAL TABLE ${0} ("아이디" int, "텍스트" text, "숫자" int) using csv location ${table.path}; \ No newline at end of file +CREATE EXTERNAL TABLE ${0} ("아이디" int, "텍스트" text, "숫자" int) using text location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/table1_ddl.sql b/tajo-core/src/test/resources/queries/TestCreateTable/table1_ddl.sql index 11ced2a063..e26975ae8b 100644 --- a/tajo-core/src/test/resources/queries/TestCreateTable/table1_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestCreateTable/table1_ddl.sql @@ -1 +1 @@ -CREATE EXTERNAL TABLE ${0} (id int, str text, num int) using csv location ${table.path}; \ No newline at end of file +CREATE EXTERNAL TABLE ${0} (id int, str text, num int) using text location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql index c3c67c7bf9..3582403d1f 100644 --- a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql @@ -1 +1 @@ -insert overwrite into location '/tajo-data/testInsertOverwriteLocationWithCompression' USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') select * from default.lineitem where l_orderkey = 3; \ No newline at end of file +insert overwrite into location '/tajo-data/testInsertOverwriteLocationWithCompression' USING text WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') select * from default.lineitem where l_orderkey = 3; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql index dd49f84034..8e76b9e6d4 100644 --- a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql @@ -1,4 +1,4 @@ create table testInsertOverwriteWithCompression (col1 int4, col2 int4, col3 float8) -USING csv -WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec'); \ No newline at end of file +USING text +WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec'); \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/create_customer_large_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/create_customer_large_ddl.sql index e5be236ff8..a9122f3dd5 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/create_customer_large_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/create_customer_large_ddl.sql @@ -4,4 +4,4 @@ create external table if not exists customer_large ( c_custkey INT4, c_name TEXT, c_address TEXT, c_nationkey INT4, c_phone TEXT, c_acctbal FLOAT8, c_mktsegment TEXT, c_comment TEXT) -using csv with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; \ No newline at end of file +using text with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/create_lineitem_large_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/create_lineitem_large_ddl.sql index 366f22bd34..f8d574f2e2 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/create_lineitem_large_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/create_lineitem_large_ddl.sql @@ -4,4 +4,4 @@ create external table if not exists lineitem_large ( l_orderkey INT4, l_partkey INT4, l_suppkey INT4, l_linenumber INT4, l_quantity FLOAT8, l_extendedprice FLOAT8, l_discount FLOAT8, l_tax FLOAT8, l_returnflag TEXT, l_linestatus TEXT, l_shipdate TEXT, l_commitdate TEXT, l_receiptdate TEXT, l_shipinstruct TEXT, l_shipmode TEXT, l_comment TEXT) -using csv with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; \ No newline at end of file +using text with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/create_orders_large_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/create_orders_large_ddl.sql index 4b9bb65f49..8f99e5e3a5 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/create_orders_large_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/create_orders_large_ddl.sql @@ -4,4 +4,4 @@ create external table if not exists orders_large ( o_orderkey INT4, o_custkey INT4, o_orderstatus TEXT, o_totalprice FLOAT8, o_orderdate TEXT, o_orderpriority TEXT, o_clerk TEXT, o_shippriority INT4, o_comment TEXT) -using csv with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; \ No newline at end of file +using text with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql index faf66f22d5..454581abf7 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql @@ -1,3 +1,3 @@ -create external table testOuterJoinAndCaseWhen1 (id int, name text, score float, type text) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; +create external table testOuterJoinAndCaseWhen1 (id int, name text, score float, type text) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql index 5680663890..ac1e1f662b 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql @@ -1,3 +1,3 @@ -create external table testOuterJoinAndCaseWhen2 (id int, name text, score float, type text) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; +create external table testOuterJoinAndCaseWhen2 (id int, name text, score float, type text) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql index 0d35ceece1..c991f71a70 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql @@ -1,3 +1,3 @@ -create external table ${0} (id int, name text, score float, type text) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; +create external table ${0} (id int, name text, score float, type text) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql index 3a7a44aefa..8f0b8de2b4 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql @@ -1,3 +1,3 @@ -create external table ${0} (id bigint, name text, score float, type text) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; +create external table ${0} (id bigint, name text, score float, type text) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestNetTypes/table1_ddl.sql b/tajo-core/src/test/resources/queries/TestNetTypes/table1_ddl.sql index c3ee88e094..b4cc87e5f2 100644 --- a/tajo-core/src/test/resources/queries/TestNetTypes/table1_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestNetTypes/table1_ddl.sql @@ -1,4 +1,4 @@ -- It is used in TestNetTypes -create external table IF NOT EXISTS table1 (id int, name text, score float, type text, addr inet4) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; \ No newline at end of file +create external table IF NOT EXISTS table1 (id int, name text, score float, type text, addr inet4) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestNetTypes/table2_ddl.sql b/tajo-core/src/test/resources/queries/TestNetTypes/table2_ddl.sql index 30bfafe4a0..40f0464174 100644 --- a/tajo-core/src/test/resources/queries/TestNetTypes/table2_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestNetTypes/table2_ddl.sql @@ -1,4 +1,4 @@ -- It is used in TestNetTypes -create external table IF NOT EXISTS table2 (id int, name text, score float, type text, addr inet4) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; \ No newline at end of file +create external table IF NOT EXISTS table2 (id int, name text, score float, type text, addr inet4) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table1_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table1_ddl.sql index 2b4a2ce29c..8309d119ef 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table1_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table1_ddl.sql @@ -1,3 +1,3 @@ -create external table table1 (id int, name text, score float, type text) using csv -with ('csvfile.delimiter'='||', 'csvfile.null'='NULL') location ${table.path}; +create external table table1 (id int, name text, score float, type text) using text +with ('text.delimiter'='||', 'text.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table2_ddl.sql index d918ac6aac..2fb821aba5 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table2_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/multibytes_delimiter_table2_ddl.sql @@ -1,3 +1,3 @@ -create external table table2 (id int, name text, score float, type text) using csv -with ('csvfile.delimiter'='ㅎ', 'csvfile.null'='NULL') location ${table.path}; +create external table table2 (id int, name text, score float, type text) using text +with ('text.delimiter'='ㅎ', 'text.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_asc_desc_keys.sql b/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_asc_desc_keys.sql index 3ffa2b305d..936e5edc36 100644 --- a/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_asc_desc_keys.sql +++ b/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_asc_desc_keys.sql @@ -1 +1 @@ -create external table table2 (col1 int8, col2 int8) using csv with ('csvfile.delimiter'=',') location ${table.path}; \ No newline at end of file +create external table table2 (col1 int8, col2 int8) using text with ('text.delimiter'=',') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql b/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql index 322b35cfb9..3ba63e0cd7 100644 --- a/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestSortQuery/create_table_with_date_ddl.sql @@ -5,6 +5,6 @@ create external table testSortWithDate ( col1 timestamp, col2 date, col3 time -) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') +) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql b/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql index 77e76d55c5..ac49b23a55 100644 --- a/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestTablePartitions/lineitemspecial_ddl.sql @@ -1,3 +1,3 @@ create external table if not exists lineitemspecial ( l_orderkey INT4, l_shipinstruct TEXT, l_shipmode TEXT) -using csv with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; +using text with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql b/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql index 846cbb63be..caeeaf9860 100644 --- a/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql @@ -5,6 +5,6 @@ create external table table1 ( col1 timestamp, col2 date, col3 time -) using csv -with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') +) using text +with ('text.delimiter'='|', 'text.null'='NULL') location ${table.path}; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestTruncateTable/table2_ddl.sql b/tajo-core/src/test/resources/queries/TestTruncateTable/table2_ddl.sql index 8d7fba023b..f00f5933d1 100644 --- a/tajo-core/src/test/resources/queries/TestTruncateTable/table2_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestTruncateTable/table2_ddl.sql @@ -1 +1 @@ -CREATE EXTERNAL TABLE ${0} (id int, str text, num int) using csv location ${table.path}; +CREATE EXTERNAL TABLE ${0} (id int, str text, num int) using text location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/default/create_table_12.sql b/tajo-core/src/test/resources/queries/default/create_table_12.sql index 0816084cc5..99e7b8d789 100644 --- a/tajo-core/src/test/resources/queries/default/create_table_12.sql +++ b/tajo-core/src/test/resources/queries/default/create_table_12.sql @@ -1,3 +1,3 @@ create external table table1 (name text, age int) -USING csv WITH ('csvfile.delimiter'='|') +USING text WITH ('text.delimiter'='|') location '/user/hive/table1' \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testDescTable.result b/tajo-core/src/test/resources/results/TestTajoCli/testDescTable.result index d3800ab183..f065e6e4db 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testDescTable.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testDescTable.result @@ -2,7 +2,7 @@ OK table name: default.TEST_DESC_TABLE table uri: ${table.path} -store type: CSV +store type: TEXT number of rows: 0 volume: 0 B Options: @@ -16,7 +16,7 @@ col2 INT4 table name: default.TEST_DESC_TABLE table uri: ${table.path} -store type: CSV +store type: TEXT number of rows: 0 volume: 0 B Options: diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result b/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result index 7eff4afacd..29106b3528 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result @@ -2,7 +2,7 @@ OK table name: default.TEST_DESC_TABLE_NESTED table uri: ${table.path} -store type: CSV +store type: TEXT number of rows: 0 volume: 0 B Options: @@ -17,7 +17,7 @@ col3 RECORD (col4 RECORD (col5 TEXT)) table name: default.TEST_DESC_TABLE_NESTED table uri: ${table.path} -store type: CSV +store type: TEXT number of rows: 0 volume: 0 B Options: diff --git a/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result b/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result index 326ba3fcda..8accecede1 100644 --- a/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result +++ b/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result @@ -10,7 +10,7 @@ CREATE DATABASE IF NOT EXISTS "TestTajoDump"; -- --- Name: "TestTajoDump"."TableName1"; Type: TABLE; Storage: CSV +-- Name: "TestTajoDump"."TableName1"; Type: TABLE; Storage: TEXT -- -CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING CSV WITH ('text.delimiter'='|'); +CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|'); diff --git a/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result index 6c15e3eace..787562e6c8 100644 --- a/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result +++ b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result @@ -10,7 +10,7 @@ CREATE DATABASE IF NOT EXISTS "TestTajoDump"; -- --- Name: "TestTajoDump"."TableName2"; Type: TABLE; Storage: CSV +-- Name: "TestTajoDump"."TableName2"; Type: TABLE; Storage: TEXT -- -CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING CSV WITH ('text.delimiter'='|'); +CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING TEXT WITH ('text.delimiter'='|'); diff --git a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result index b1172a778c..7e10a3bfba 100644 --- a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result +++ b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result @@ -1,4 +1,4 @@ -- --- Name: db1.table2; Type: TABLE; Storage: CSV +-- Name: db1.table2; Type: TABLE; Storage: TEXT -- -CREATE TABLE db1.table2 (name BLOB, addr TEXT) USING CSV WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|'); \ No newline at end of file +CREATE TABLE db1.table2 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|'); \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result index a5b5f63cad..535bc11827 100644 --- a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result +++ b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result @@ -1,5 +1,5 @@ -- --- Name: db1.table1; Type: TABLE; Storage: CSV +-- Name: db1.table1; Type: TABLE; Storage: TEXT -- Path: /table1 -- -CREATE EXTERNAL TABLE db1.table1 (name BLOB, addr TEXT) USING CSV WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN(key INT4, key2 TEXT) LOCATION '/table1'; \ No newline at end of file +CREATE EXTERNAL TABLE db1.table1 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN(key INT4, key2 TEXT) LOCATION '/table1'; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result index 43be915dc5..34b6fb361c 100644 --- a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result +++ b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result @@ -1,5 +1,5 @@ -- --- Name: db1."TABLE2"; Type: TABLE; Storage: CSV +-- Name: db1."TABLE2"; Type: TABLE; Storage: TEXT -- Path: /table1 -- -CREATE EXTERNAL TABLE db1."TABLE2" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING CSV WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN("BirthYear" INT4) LOCATION '/table1'; \ No newline at end of file +CREATE EXTERNAL TABLE db1."TABLE2" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN("BirthYear" INT4) LOCATION '/table1'; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result index 764892f862..cd1ebf874d 100644 --- a/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result +++ b/tajo-core/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result @@ -1,4 +1,4 @@ -- --- Name: db1."TABLE1"; Type: TABLE; Storage: CSV +-- Name: db1."TABLE1"; Type: TABLE; Storage: TEXT -- -CREATE TABLE db1."TABLE1" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING CSV WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN("BirthYear" INT4); \ No newline at end of file +CREATE TABLE db1."TABLE1" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN("BirthYear" INT4); \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/backup_and_restore/catalog.rst b/tajo-docs/src/main/sphinx/backup_and_restore/catalog.rst index 1c2b7096b6..5676b1732b 100644 --- a/tajo-docs/src/main/sphinx/backup_and_restore/catalog.rst +++ b/tajo-docs/src/main/sphinx/backup_and_restore/catalog.rst @@ -25,7 +25,7 @@ For example, if you want to backup a table customer, you should type a command a -- -- - -- Name: customer; Type: TABLE; Storage: CSV + -- Name: customer; Type: TABLE; Storage: TEXT -- Path: file:/home/hyunsik/tpch/customer -- CREATE EXTERNAL TABLE customer (c_custkey INT8, c_name TEXT, c_address TEXT, c_nationkey INT8, c_phone TEXT, c_acctbal FLOAT8, c_mktsegment TEXT, c_comment TEXT) USING TEXT LOCATION 'file:/home/hyunsik/tpch/customer'; diff --git a/tajo-docs/src/main/sphinx/getting_started.rst b/tajo-docs/src/main/sphinx/getting_started.rst index e30c3fef28..ecb013eebb 100644 --- a/tajo-docs/src/main/sphinx/getting_started.rst +++ b/tajo-docs/src/main/sphinx/getting_started.rst @@ -152,7 +152,7 @@ If you want to know DDL statements in more detail, please see Query Language. :: table name: table1 table path: file:/home/x/table1 - store type: CSV + store type: TEXT number of rows: 0 volume (bytes): 78 B schema: diff --git a/tajo-docs/src/main/sphinx/index/how_to_use.rst b/tajo-docs/src/main/sphinx/index/how_to_use.rst index 776d205733..09e1b725c9 100644 --- a/tajo-docs/src/main/sphinx/index/how_to_use.rst +++ b/tajo-docs/src/main/sphinx/index/how_to_use.rst @@ -18,7 +18,7 @@ If the index is created successfully, you can see the information about that ind table name: default.lineitem table path: hdfs://localhost:7020/tpch/lineitem - store type: CSV + store type: TEXT number of rows: unknown volume: 753.9 MB Options: diff --git a/tajo-docs/src/main/sphinx/tsql/meta_command.rst b/tajo-docs/src/main/sphinx/tsql/meta_command.rst index 057124d87e..7d687a12f2 100644 --- a/tajo-docs/src/main/sphinx/tsql/meta_command.rst +++ b/tajo-docs/src/main/sphinx/tsql/meta_command.rst @@ -80,7 +80,7 @@ Basic usages table name: orders table path: hdfs:/xxx/xxx/tpch/orders - store type: CSV + store type: TEXT number of rows: 0 volume (bytes): 172.0 MB schema: diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index ef8663d35b..90db1ad331 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.OverridableConf; import org.apache.tajo.QueryVars; import org.apache.tajo.SessionVars; @@ -1781,7 +1782,7 @@ private InsertNode buildInsertIntoLocationPlan(PlanContext context, InsertNode i insertNode.setUri(targetUri); if (expr.hasStorageType()) { - insertNode.setStorageType(expr.getStorageType()); + insertNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(expr.getStorageType())); } if (expr.hasParams()) { KeyValueSet options = new KeyValueSet(); @@ -1826,7 +1827,8 @@ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, createTableNode.setTableSchema(parentTableDesc.getSchema()); createTableNode.setPartitionMethod(partitionDesc); - createTableNode.setStorageType(parentTableDesc.getMeta().getStoreType()); + createTableNode.setStorageType( + CatalogUtil.getBackwardCompitablityStoreType(parentTableDesc.getMeta().getStoreType())); createTableNode.setOptions(parentTableDesc.getMeta().getOptions()); createTableNode.setExternal(parentTableDesc.isExternal()); @@ -1862,9 +1864,9 @@ public LogicalNode visitCreateTable(PlanContext context, Stack stack, Crea createTableNode.setUri(getCreatedTableURI(context, expr)); if (expr.hasStorageType()) { // If storage type (using clause) is specified - createTableNode.setStorageType(expr.getStorageType()); + createTableNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(expr.getStorageType())); } else { // otherwise, default type - createTableNode.setStorageType("CSV"); + createTableNode.setStorageType(BuiltinStorages.TEXT); } // Set default storage properties to table diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java index 55768898a2..116cb15cee 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java @@ -41,7 +41,7 @@ public static void setTableProperty(OverridableConf context, CreateTableNode nod String storeType = node.getStorageType(); KeyValueSet property = node.getOptions(); - if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("TEXT")) { + if (storeType.equalsIgnoreCase("TEXT") || storeType.equalsIgnoreCase("TEXT")) { setSessionToProperty(context, SessionVars.NULL_CHAR, property, StorageConstants.TEXT_NULL); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java index 19a18201d2..c892eda0a2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java @@ -31,7 +31,7 @@ * This includes some basic information for materializing data. */ public abstract class PersistentStoreNode extends UnaryNode implements Cloneable { - @Expose protected String storageType = "CSV"; + @Expose protected String storageType = "TEXT"; @Expose protected KeyValueSet options = new KeyValueSet(); protected PersistentStoreNode(int pid, NodeType nodeType) { diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java index 694eeeb18d..92cb39952a 100644 --- a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java @@ -44,13 +44,13 @@ public void testEquals() { GroupbyNode groupbyNode = new GroupbyNode(0); groupbyNode.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)}); ScanNode scanNode = new ScanNode(0); - scanNode.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("CSV"), new Path("in"))); + scanNode.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("TEXT"), new Path("in"))); GroupbyNode groupbyNode2 = new GroupbyNode(0); groupbyNode2.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)}); JoinNode joinNode = new JoinNode(0); ScanNode scanNode2 = new ScanNode(0); - scanNode2.init(CatalogUtil.newTableDesc("in2", schema, CatalogUtil.newTableMeta("CSV"), new Path("in2"))); + scanNode2.init(CatalogUtil.newTableDesc("in2", schema, CatalogUtil.newTableMeta("TEXT"), new Path("in2"))); groupbyNode.setChild(scanNode); groupbyNode2.setChild(joinNode); @@ -61,7 +61,7 @@ public void testEquals() { assertFalse(groupbyNode.deepEquals(groupbyNode2)); ScanNode scanNode3 = new ScanNode(0); - scanNode3.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("CSV"), new Path("in"))); + scanNode3.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("TEXT"), new Path("in"))); groupbyNode2.setChild(scanNode3); assertTrue(groupbyNode.equals(groupbyNode2)); diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java index 5961751249..2d3f2c3096 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -23,11 +23,10 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.hbase.StorageFragmentProtos.*; +import org.apache.tajo.storage.hbase.StorageFragmentProtos.HBaseFragmentProto; import java.net.URI; @@ -173,7 +172,7 @@ public FragmentProto getProto() { FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(this.tableName); fragmentBuilder.setContents(builder.buildPartial().toByteString()); - fragmentBuilder.setStoreType(CatalogUtil.getStoreTypeString(StoreType.HBASE)); + fragmentBuilder.setStoreType(BuiltinStorages.HBASE); return fragmentBuilder.build(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index ab1c808865..5f94aae554 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -24,9 +24,8 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; -import org.apache.tajo.storage.StorageFragmentProtos.*; +import org.apache.tajo.BuiltinStorages; +import org.apache.tajo.storage.StorageFragmentProtos.FileFragmentProto; import org.apache.tajo.util.TUtil; import java.io.IOException; @@ -231,7 +230,7 @@ public FragmentProto getProto() { FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(this.tableName); - fragmentBuilder.setStoreType(CatalogUtil.getStoreTypeString(StoreType.CSV)); + fragmentBuilder.setStoreType(BuiltinStorages.TEXT); fragmentBuilder.setContents(builder.buildPartial().toByteString()); return fragmentBuilder.build(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index 9740b4dea4..67d0646e31 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -68,10 +68,9 @@ public TestCompressionStorages(String type) throws IOException { @Parameterized.Parameters public static Collection generateParameters() { return Arrays.asList(new Object[][]{ - {"CSV"}, + {"TEXT"}, {"RCFILE"}, - {"SEQUENCEFILE"}, - {"TEXT"} + {"SEQUENCEFILE"} }); } @@ -153,14 +152,6 @@ private void storageCompressionTest(String storeType, Class splits = Lists.newArrayList(); // Get FileFragments in partition batch @@ -193,7 +193,7 @@ public void testGetSplitWithBlockStorageLocationsBatching() throws Exception { schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.INT4); schema.addColumn("name", Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); List splits = Lists.newArrayList(); splits.addAll(sm.getSplits("data", meta, schema, tablePath)); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index b53dbeccd0..506d430b9f 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -135,7 +135,6 @@ public TestStorages(String type, boolean splitable, boolean statsable, boolean s public static Collection generateParameters() { return Arrays.asList(new Object[][] { //type, splitable, statsable, seekable - {"CSV", true, true, true}, {"RAW", false, true, true}, {"RCFILE", true, true, false}, {"PARQUET", false, false, false}, @@ -776,7 +775,7 @@ public void testSequenceFileBinarySerializeDeserialize() throws IOException { @Test public void testTime() throws IOException { - if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("RAW")) { + if (storeType.equalsIgnoreCase("TEXT") || storeType.equalsIgnoreCase("RAW")) { Schema schema = new Schema(); schema.addColumn("col1", Type.DATE); schema.addColumn("col2", Type.TIME); @@ -1021,7 +1020,7 @@ public void testLessThanSchemaSize() throws IOException { @Test public final void testInsertFixedCharTypeWithOverSize() throws Exception { - if (storeType.equalsIgnoreCase("CSV") == false && + if (storeType.equalsIgnoreCase("TEXT") == false && storeType.equalsIgnoreCase("SEQUENCEFILE") == false && storeType.equalsIgnoreCase("RCFILE") == false && storeType.equalsIgnoreCase("PARQUET") == false) { diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java index 22fb607549..ed010c090a 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java @@ -72,7 +72,6 @@ public TestBSTIndex(String type) { @Parameterized.Parameters public static Collection generateParameters() { return Arrays.asList(new Object[][]{ - {"CSV"}, {"RAW"}, {"TEXT"} }); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index fac7d7751b..c198965aa3 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.*; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -71,7 +72,7 @@ public void setUp() throws Exception { @Test public void testFindValueInSingleCSV() throws IOException { - meta = CatalogUtil.newTableMeta("CSV"); + meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv"); fs.mkdirs(tablePath.getParent()); @@ -160,7 +161,7 @@ public void testFindValueInSingleCSV() throws IOException { @Test public void testFindNextKeyValueInSingleCSV() throws IOException { - meta = CatalogUtil.newTableMeta("CSV"); + meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV", "table1.csv"); From 8d38644195ed0c1eca28f3df4521b089d656a398 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Tue, 21 Jul 2015 18:11:22 +0900 Subject: [PATCH 3/3] fix hive meta failure --- .../java/org/apache/tajo/catalog/store/HiveCatalogStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index d593e4c445..8a08b77a3e 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -489,7 +489,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); } - } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.TEXTFILE)) { + } else if (tableDesc.getMeta().getStoreType().equals(BuiltinStorages.TEXT)) { sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());