diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java index d5dc64fff8c8..01710788546d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java @@ -84,25 +84,25 @@ public static ChunkMetadataCache getInstance() { /** * get {@link ChunkMetadata}. THREAD SAFE. */ - public List get(TsFileResource resource, Path seriesPath) + public List get(String filePath, Path seriesPath) throws IOException { if (!cacheEnable) { - TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(resource); // bloom filter part + TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(filePath); BloomFilter bloomFilter = fileMetaData.getBloomFilter(); if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) { if (logger.isDebugEnabled()) { logger.debug(String - .format("path not found by bloom filter, file is: %s, path is: %s", resource.getFile(), seriesPath)); + .format("path not found by bloom filter, file is: %s, path is: %s", filePath, seriesPath)); } return new ArrayList<>(); } // If timeseries isn't included in the tsfile, empty list is returned. - TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(resource.getPath(), true); + TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); return tsFileReader.getChunkMetadataList(seriesPath); } - String key = (resource.getPath() + IoTDBConstant.PATH_SEPARATOR + String key = (filePath + IoTDBConstant.PATH_SEPARATOR + seriesPath.getDevice() + seriesPath.getMeasurement()).intern(); cacheRequestNum.incrementAndGet(); @@ -126,14 +126,14 @@ public List get(TsFileResource resource, Path seriesPath) return new ArrayList<>(lruCache.get(key)); } printCacheLog(false); - TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(resource); // bloom filter part + TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(filePath); BloomFilter bloomFilter = fileMetaData.getBloomFilter(); if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) { return new ArrayList<>(); } List chunkMetaDataList = FileLoaderUtils - .getChunkMetadataList(seriesPath, resource); + .getChunkMetadataList(seriesPath, filePath); lruCache.put(key, chunkMetaDataList); return chunkMetaDataList; } finally { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index f61572415afe..3ffc7fa3c686 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -20,13 +20,15 @@ package org.apache.iotdb.db.engine.cache; import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.utils.BloomFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.Map; import java.util.Objects; @@ -82,6 +84,13 @@ public static TimeSeriesMetadataCache getInstance() { public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set allSensors) throws IOException { if (!cacheEnable) { + // bloom filter part + TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(key.filePath); + BloomFilter bloomFilter = fileMetaData.getBloomFilter(); + if (bloomFilter != null && !bloomFilter + .contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { + return null; + } TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); return reader.readDeviceMetadata(key.device).get(key.measurement); } @@ -107,6 +116,13 @@ public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set allSen return lruCache.get(key); } printCacheLog(false); + // bloom filter part + TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(key.filePath); + BloomFilter bloomFilter = fileMetaData.getBloomFilter(); + if (bloomFilter != null && !bloomFilter + .contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { + return null; + } TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); Map timeSeriesMetadataMap = reader.readDeviceMetadata(key.device); TimeseriesMetadata res = timeSeriesMetadataMap.get(key.measurement); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java index bf4314979b00..6870e53492c9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java @@ -90,22 +90,21 @@ public static TsFileMetaDataCache getInstance() { /** * get the TsFileMetaData for given TsFile. * - * @param tsFileResource -given TsFile + * @param filePath -given TsFile */ - public TsFileMetadata get(TsFileResource tsFileResource) throws IOException { + public TsFileMetadata get(String filePath) throws IOException { if (!cacheEnable) { - return FileLoaderUtils.getTsFileMetadata(tsFileResource); + return FileLoaderUtils.getTsFileMetadata(filePath); } - String path = tsFileResource.getPath().intern(); cacheRequestNum.incrementAndGet(); lock.readLock().lock(); try { - if (cache.containsKey(path)) { + if (cache.containsKey(filePath)) { cacheHitNum.incrementAndGet(); printCacheLog(true); - return cache.get(path); + return cache.get(filePath); } } finally { lock.readLock().unlock(); @@ -113,14 +112,14 @@ public TsFileMetadata get(TsFileResource tsFileResource) throws IOException { lock.writeLock().lock(); try { - if (cache.containsKey(tsFileResource.getPath())) { + if (cache.containsKey(filePath)) { cacheHitNum.incrementAndGet(); printCacheLog(true); - return cache.get(tsFileResource.getPath()); + return cache.get(filePath); } printCacheLog(false); - TsFileMetadata fileMetaData = FileLoaderUtils.getTsFileMetadata(tsFileResource); - cache.put(tsFileResource.getPath(), fileMetaData); + TsFileMetadata fileMetaData = FileLoaderUtils.getTsFileMetadata(filePath); + cache.put(filePath, fileMetaData); return fileMetaData; } finally { lock.writeLock().unlock(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index be13ba27c659..ce2d7daf720e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -34,7 +34,6 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.io.File; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java index 0644627dcc1d..d7a488f8b7c6 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java @@ -22,7 +22,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.expression.IExpression; - import java.util.*; public class RawDataQueryPlan extends QueryPlan { diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java index 8bbadf23e5c7..2d4d555cd51a 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java @@ -21,7 +21,6 @@ import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_VALUE; import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TIMESERIES; - import org.apache.iotdb.db.engine.querycontext.QueryDataSource; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; @@ -29,7 +28,6 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.MManager; import org.apache.iotdb.db.metadata.mnode.LeafMNode; -import org.apache.iotdb.db.metadata.mnode.MNode; import org.apache.iotdb.db.qp.physical.crud.LastQueryPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.QueryResourceManager; @@ -45,7 +43,6 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; - import java.io.IOException; import java.util.*; @@ -66,9 +63,8 @@ public LastQueryExecutor(LastQueryPlan lastQueryPlan) { public QueryDataSet execute(QueryContext context) throws StorageEngineException, IOException, QueryProcessException { - ListDataSet dataSet = - new ListDataSet( - Arrays.asList(new Path(COLUMN_TIMESERIES), new Path(COLUMN_VALUE)), + ListDataSet dataSet = new ListDataSet( + Arrays.asList(new Path(COLUMN_TIMESERIES), new Path(COLUMN_VALUE)), Arrays.asList(TSDataType.TEXT, TSDataType.TEXT)); for (int i = 0; i < selectedSeries.size(); i++) { @@ -122,14 +118,12 @@ private TimeValuePair calculateLastPairForOneSeries( if (!seqFileResources.isEmpty()) { for (int i = seqFileResources.size() - 1; i >= 0; i--) { - List chunkMetadata = - FileLoaderUtils.loadChunkMetadataFromTsFileResource( + List chunkMetadata = FileLoaderUtils.loadChunkMetadataFromTsFileResource( seqFileResources.get(i), seriesPath, context); if (!chunkMetadata.isEmpty()) { ChunkMetadata lastChunkMetaData = chunkMetadata.get(chunkMetadata.size() - 1); Statistics chunkStatistics = lastChunkMetaData.getStatistics(); - resultPair = - constructLastPair( + resultPair = constructLastPair( chunkStatistics.getEndTime(), chunkStatistics.getLastValue(), tsDataType); break; } @@ -147,8 +141,7 @@ private TimeValuePair calculateLastPairForOneSeries( if (chunkMetaData.getEndTime() == resultPair.getTimestamp() && chunkMetaData.getVersion() > version) { Statistics chunkStatistics = chunkMetaData.getStatistics(); - resultPair = - constructLastPair( + resultPair = constructLastPair( chunkStatistics.getEndTime(), chunkStatistics.getLastValue(), tsDataType); version = chunkMetaData.getVersion(); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java index cbd977f942ff..625daacb7685 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java @@ -39,7 +39,6 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.iotdb.tsfile.read.query.timegenerator.TimeGenerator; - import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java index a5ec31b83714..add6e6b566e2 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java @@ -30,7 +30,6 @@ import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; - import java.io.IOException; import java.util.List; @@ -51,7 +50,7 @@ public DiskChunkMetadataLoader(TsFileResource resource, Path seriesPath, QueryCo @Override public List getChunkMetadataList() throws IOException { List chunkMetadataList = ChunkMetadataCache - .getInstance().get(resource, seriesPath); + .getInstance().get(resource.getPath(), seriesPath); setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 52b7214a7470..2ae5a099a0b8 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@ -42,7 +42,6 @@ import org.apache.iotdb.tsfile.read.reader.IChunkReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; - import java.io.IOException; import java.util.*; import java.util.stream.Collectors; diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index 303afa6a8bff..ae2717201053 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -34,7 +34,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.filter.basic.Filter; - import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -128,7 +127,7 @@ public static List loadChunkMetadataFromTsFileResource( return new ArrayList<>(); } if (resource.isClosed()) { - chunkMetadataList = ChunkMetadataCache.getInstance().get(resource, seriesPath); + chunkMetadataList = ChunkMetadataCache.getInstance().get(resource.getPath(), seriesPath); } else { chunkMetadataList = resource.getChunkMetadataList(); } @@ -161,19 +160,13 @@ public static List loadChunkMetadataFromTsFileResource( return chunkMetadataList; } - public static List getChunkMetadataList(Path path, TsFileResource resource) throws IOException { - if (!resource.isClosed()) { - throw new IOException("The TsFile is not closed: " + resource.getFile().getAbsolutePath()); - } - TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(resource.getPath(), true); + public static List getChunkMetadataList(Path path, String filePath) throws IOException { + TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); return tsFileReader.getChunkMetadataList(path); } - public static TsFileMetadata getTsFileMetadata(TsFileResource resource) throws IOException { - if (!resource.isClosed()) { - throw new IOException("The TsFile is not closed: " + resource.getFile().getAbsolutePath()); - } - TsFileSequenceReader reader = FileReaderManager.getInstance().get(resource.getPath(), true); + public static TsFileMetadata getTsFileMetadata(String filePath) throws IOException { + TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true); return reader.readFileMetadata(); } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java index 55660420167b..292697394dd3 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java @@ -67,16 +67,6 @@ public void setUp() throws Exception { MetadataManagerHelper.initMetadata(); ActiveTimeSeriesCounter.getInstance().init(storageGroup); storageGroupProcessor = new StorageGroupProcessor(systemDir, storageGroup, new DirectFlushPolicy()); -// MManager.getInstance().createTimeseries(storageGroup + "." + measurementId0, -// new MeasurementSchema(measurementId0, TSDataType.INT32, TSEncoding.PLAIN)); -// MManager.getInstance().createTimeseries(storageGroup + "." + measurementId0, -// new MeasurementSchema(measurementId0, TSDataType.INT64, TSEncoding.PLAIN)); -// MManager.getInstance().createTimeseries(storageGroup + "." + measurementId0, -// new MeasurementSchema(measurementId0, TSDataType.FLOAT, TSEncoding.PLAIN)); -// MManager.getInstance().createTimeseries(storageGroup + "." + measurementId0, -// new MeasurementSchema(measurementId0, TSDataType.DOUBLE, TSEncoding.PLAIN)); -// MManager.getInstance().createTimeseries(storageGroup + "." + measurementId0, -// new MeasurementSchema(measurementId0, TSDataType.BOOLEAN, TSEncoding.PLAIN)); insertData(); } @@ -145,7 +135,7 @@ public void test1() throws IOException, QueryProcessException { Assert.assertFalse(unseqResources.get(3).isClosed()); List metaDataList = ChunkMetadataCache.getInstance() - .get(seqResources.get(0), new Path(storageGroup, measurementId5)); + .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5)); Assert.assertEquals(0, metaDataList.size()); } @@ -167,7 +157,7 @@ public void test2() throws IOException, QueryProcessException { Assert.assertFalse(unseqResources.get(3).isClosed()); List metaDataList = ChunkMetadataCache.getInstance() - .get(seqResources.get(0), new Path(storageGroup, measurementId5)); + .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5)); Assert.assertEquals(0, metaDataList.size()); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java index abf45d9e46f4..feb147e4b5fd 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java @@ -26,7 +26,6 @@ import org.apache.iotdb.tsfile.utils.BloomFilter; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; - import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer;