diff --git a/pom.xml b/pom.xml index 57e0d39b03242..bbf972d364fb7 100644 --- a/pom.xml +++ b/pom.xml @@ -44,8 +44,6 @@ ssh://git@github.com:apache/incubator-iotdb.git rel/0.10 - - @@ -53,12 +51,10 @@ scm:git:https://gitbox.apache.org/repos/asf/incubator-iotdb-website.git - Jira https://issues.apache.org/jira/browse/iotdb - Apache IoTDB Developer List @@ -82,7 +78,6 @@ http://mail-archives.apache.org/mod_mbox/iotdb-notifications/ - tsfile service-rpc diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/ChunkReaderWrap.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/ChunkReaderWrap.java index 1d3643ab359f7..d196960a2fb71 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/ChunkReaderWrap.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/ChunkReaderWrap.java @@ -71,7 +71,7 @@ public ChunkReaderWrap(ReadOnlyMemChunk readOnlyMemChunk, Filter filter) { public IPointReader getIPointReader() throws IOException { if (type.equals(ChunkReaderType.DISK_CHUNK)) { - Chunk chunk = chunkLoader.getChunk(chunkMetaData); + Chunk chunk = chunkLoader.loadChunk(chunkMetaData); ChunkReader chunkReader = new ChunkReader(chunk, filter); return new ChunkDataIterator(chunkReader); } else { @@ -81,7 +81,7 @@ public IPointReader getIPointReader() throws IOException { public IReaderByTimestamp getIReaderByTimestamp() throws IOException { if (type.equals(ChunkReaderType.DISK_CHUNK)) { - Chunk chunk = chunkLoader.getChunk(chunkMetaData); + Chunk chunk = chunkLoader.loadChunk(chunkMetaData); ChunkReaderByTimestamp chunkReader = new ChunkReaderByTimestamp(chunk); return new DiskChunkReaderByTimestamp(chunkReader); } else { diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java index daac24a3b2155..11d6c15d4e44b 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java @@ -27,6 +27,9 @@ import java.io.IOException; +/** + * To read one chunk from disk, and only used in iotdb server module + */ public class DiskChunkLoader implements IChunkLoader { private TsFileSequenceReader reader; @@ -36,7 +39,7 @@ public DiskChunkLoader(TsFileSequenceReader reader) { } @Override - public Chunk getChunk(ChunkMetadata chunkMetaData) throws IOException { + public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException { return ChunkCache.getInstance().get(chunkMetaData, reader); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkLoader.java index 0b14147072bc9..705bcb9fcf2d8 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkLoader.java @@ -26,6 +26,9 @@ import java.io.IOException; +/** + * To read one chunk from memory, and only used in iotdb server module + */ public class MemChunkLoader implements IChunkLoader { private final ReadOnlyMemChunk chunk; @@ -36,7 +39,7 @@ public MemChunkLoader(ReadOnlyMemChunk chunk) { @Override - public Chunk getChunk(ChunkMetadata chunkMetaData) { + public Chunk loadChunk(ChunkMetadata chunkMetaData) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkReader.java index 7f97a85f793d0..d831063a8a409 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemChunkReader.java @@ -116,7 +116,7 @@ public void close() { } @Override - public List getPageReaderList() throws IOException { + public List loadPageReaderList() throws IOException { // we treat one ReadOnlyMemChunk as one Page return Collections.singletonList( new MemPageReader(nextPageData(), readOnlyMemChunk.getChunkMetaData().getStatistics())); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java index add6e6b566e21..42d325161cc78 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java @@ -48,7 +48,7 @@ public DiskChunkMetadataLoader(TsFileResource resource, Path seriesPath, QueryCo } @Override - public List getChunkMetadataList() throws IOException { + public List loadChunkMetadataList() throws IOException { List chunkMetadataList = ChunkMetadataCache .getInstance().get(resource.getPath(), seriesPath); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java index 7c3998d7def0a..4a54437d184eb 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java @@ -44,7 +44,7 @@ public MemChunkMetadataLoader(TsFileResource resource, Path seriesPath, QueryCon } @Override - public List getChunkMetadataList() throws IOException { + public List loadChunkMetadataList() throws IOException { List chunkMetadataList = resource.getChunkMetadataList(); DiskChunkMetadataLoader.setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 2ae5a099a0b88..1586d6fcbdf87 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@ -22,8 +22,6 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.filter.TsFileFilter; -import org.apache.iotdb.db.query.reader.chunk.MemChunkLoader; -import org.apache.iotdb.db.query.reader.chunk.MemChunkReader; import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader; import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.db.utils.QueryUtils; @@ -34,14 +32,11 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.common.BatchData; -import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter; -import org.apache.iotdb.tsfile.read.reader.IChunkReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; -import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; + import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -246,7 +241,7 @@ private void unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(long end } private void unpackOneTimeSeriesMetadata(TimeseriesMetadata timeSeriesMetadata) throws IOException { - cachedChunkMetadata.addAll(timeSeriesMetadata.getChunkMetadataList()); + cachedChunkMetadata.addAll(FileLoaderUtils.loadChunkMetadata(timeSeriesMetadata)); } boolean isChunkOverlapped() throws IOException { @@ -343,12 +338,8 @@ private void unpackAllOverlappedChunkMetadataToCachedPageReaders(long endTime, b } private void unpackOneChunkMetaData(ChunkMetadata chunkMetaData) throws IOException { - initChunkReader(chunkMetaData) - .getPageReaderList() - .forEach( - pageReader -> - cachedPageReaders.add( - new VersionPageReader(chunkMetaData.getVersion(), pageReader))); + FileLoaderUtils.loadPageReader(chunkMetaData, timeFilter) + .forEach(pageReader -> cachedPageReaders.add(new VersionPageReader(chunkMetaData.getVersion(), pageReader))); } /** @@ -531,23 +522,6 @@ private BatchData nextOverlappedPage() throws IOException { throw new IOException("No more batch data"); } - private IChunkReader initChunkReader(ChunkMetadata metaData) throws IOException { - if (metaData == null) { - throw new IOException("Can't init null chunkMeta"); - } - IChunkReader chunkReader; - IChunkLoader chunkLoader = metaData.getChunkLoader(); - if (chunkLoader instanceof MemChunkLoader) { - MemChunkLoader memChunkLoader = (MemChunkLoader) chunkLoader; - chunkReader = new MemChunkReader(memChunkLoader.getChunk(), timeFilter); - } else { - Chunk chunk = chunkLoader.getChunk(metaData); - chunkReader = new ChunkReader(chunk, timeFilter); - chunkReader.hasNextSatisfiedPage(); - } - return chunkReader; - } - private LinkedList sortUnSeqFileResources( List tsFileResources) { return tsFileResources.stream() diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index ae2717201053b..9f9cb133de748 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -26,14 +26,22 @@ import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader; +import org.apache.iotdb.db.query.reader.chunk.MemChunkLoader; +import org.apache.iotdb.db.query.reader.chunk.MemChunkReader; import org.apache.iotdb.db.query.reader.chunk.metadata.DiskChunkMetadataLoader; import org.apache.iotdb.db.query.reader.chunk.metadata.MemChunkMetadataLoader; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.read.reader.IChunkReader; +import org.apache.iotdb.tsfile.read.reader.IPageReader; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -117,6 +125,41 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(TsFileResource resource, return timeSeriesMetadata; } + /** + * load all chunk metadata of one time series in one file. + * @param timeSeriesMetadata the corresponding TimeSeriesMetadata in that file. + */ + public static List loadChunkMetadata(TimeseriesMetadata timeSeriesMetadata) throws IOException { + return timeSeriesMetadata.loadChunkMetadataList(); + } + + + /** + * load all page readers in one chunk that satisfying the timeFilter + * @param chunkMetaData the corresponding chunk metadata + * @param timeFilter it should be a TimeFilter instead of a ValueFilter + */ + public static List loadPageReader(ChunkMetadata chunkMetaData, Filter timeFilter) throws IOException { + return initChunkReader(chunkMetaData, timeFilter).loadPageReaderList(); + } + + private static IChunkReader initChunkReader(ChunkMetadata metaData, Filter timeFilter) throws IOException { + if (metaData == null) { + throw new IOException("Can't init null chunkMeta"); + } + IChunkReader chunkReader; + IChunkLoader chunkLoader = metaData.getChunkLoader(); + if (chunkLoader instanceof MemChunkLoader) { + MemChunkLoader memChunkLoader = (MemChunkLoader) chunkLoader; + chunkReader = new MemChunkReader(memChunkLoader.getChunk(), timeFilter); + } else { + Chunk chunk = chunkLoader.loadChunk(metaData); + chunkReader = new ChunkReader(chunk, timeFilter); + chunkReader.hasNextSatisfiedPage(); + } + return chunkReader; + } + /** * load all ChunkMetadatas belong to the seriesPath */ diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java index 16994dfd9cbaa..a1ae8bd733446 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java @@ -23,7 +23,6 @@ import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; -import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.version.VersionController; import org.apache.iotdb.db.exception.StorageEngineException; @@ -189,7 +188,7 @@ public void test() throws StorageGroupProcessorException, IOException { PriorityMergeReader unSeqMergeReader = new PriorityMergeReader(); int priorityValue = 1; for (ChunkMetadata chunkMetaData : metadataQuerier.getChunkMetaDataList(path)) { - Chunk chunk = chunkLoader.getChunk(chunkMetaData); + Chunk chunk = chunkLoader.loadChunk(chunkMetaData); ChunkReader chunkReader = new ChunkReader(chunk, null); unSeqMergeReader .addReader(new ChunkDataIterator(chunkReader), priorityValue); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index be0b330e8c02e..f6051bfc72594 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -113,7 +113,7 @@ public void setChunkMetadataLoader(IChunkMetadataLoader chunkMetadataLoader) { this.chunkMetadataLoader = chunkMetadataLoader; } - public List getChunkMetadataList() throws IOException { - return chunkMetadataLoader.getChunkMetadataList(); + public List loadChunkMetadataList() throws IOException { + return chunkMetadataLoader.loadChunkMetadataList(); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java index 7a2ffe288e402..b29e46ec2058c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java @@ -27,7 +27,7 @@ import java.io.IOException; /** - * Read one Chunk and cache it into a LRUCache. + * Read one Chunk and cache it into a LRUCache, only used in tsfile module. */ public class CachedChunkLoaderImpl implements IChunkLoader { @@ -59,7 +59,7 @@ public Chunk loadObjectByKey(ChunkMetadata metaData) throws IOException { } @Override - public Chunk getChunk(ChunkMetadata chunkMetaData) throws IOException { + public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException { Chunk chunk = chunkCache.get(chunkMetaData); return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType()); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkLoader.java index fe843afad4200..fee769e8e2925 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkLoader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkLoader.java @@ -18,17 +18,17 @@ */ package org.apache.iotdb.tsfile.read.controller; -import java.io.IOException; - import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.read.common.Chunk; +import java.io.IOException; + public interface IChunkLoader { /** * read all content of any chunk. */ - Chunk getChunk(ChunkMetadata chunkMetaData) throws IOException; + Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException; /** * close the file reader. diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java index b213330619da8..255448db734a4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java @@ -28,6 +28,6 @@ public interface IChunkMetadataLoader { /** * read all chunk metadata of one time series in one file. */ - List getChunkMetadataList() throws IOException; + List loadChunkMetadataList() throws IOException; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IChunkReader.java index 3a37461505842..9103f7b2705e3 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IChunkReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IChunkReader.java @@ -31,5 +31,5 @@ public interface IChunkReader { void close() throws IOException; - List getPageReaderList() throws IOException; + List loadPageReaderList() throws IOException; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java index dc9ce4bf26737..86f74eea849de 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java @@ -157,7 +157,7 @@ public ChunkHeader getChunkHeader() { } @Override - public List getPageReaderList() { + public List loadPageReaderList() { return pageReaderList; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java index b6ef473175eee..cb2ab8cec15ca 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java @@ -39,7 +39,7 @@ public FileSeriesReader(IChunkLoader chunkLoader, @Override protected void initChunkReader(ChunkMetadata chunkMetaData) throws IOException { - Chunk chunk = chunkLoader.getChunk(chunkMetaData); + Chunk chunk = chunkLoader.loadChunk(chunkMetaData); this.chunkReader = new ChunkReader(chunk, filter); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java index 9325465dda33a..81aced259e08b 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java @@ -146,7 +146,7 @@ private boolean constructNextSatisfiedChunkReader() throws IOException { } private void initChunkReader(ChunkMetadata chunkMetaData) throws IOException { - Chunk chunk = chunkLoader.getChunk(chunkMetaData); + Chunk chunk = chunkLoader.loadChunk(chunkMetaData); this.chunkReader = new ChunkReaderByTimestamp(chunk); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java index eb32a2ebc4e9e..a4202bcf2cee4 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java @@ -57,7 +57,7 @@ public void test() throws IOException { CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader); for (ChunkMetadata chunkMetaData : chunkMetadataList) { - Chunk chunk = seriesChunkLoader.getChunk(chunkMetaData); + Chunk chunk = seriesChunkLoader.loadChunk(chunkMetaData); ChunkHeader chunkHeader = chunk.getHeader(); Assert.assertEquals(chunkHeader.getDataSize(), chunk.getData().remaining()); }