[To rel/0.13][IOTDB-2835]Fix empty page in selfcheck method of TsFile…

…SequenceReader (#5641)
apache · Apr 22, 2022 · d155692 · d155692
1 parent 06af131
commit d155692
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 11 deletions.
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -159,6 +159,7 @@
 import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
 import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
 import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
 import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
 import org.apache.iotdb.tsfile.read.common.Field;
 import org.apache.iotdb.tsfile.read.common.Path;
@@ -1286,7 +1287,12 @@ private void loadFile(File file, OperateFilePlan plan) throws QueryProcessExcept
 
       List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
       try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), false)) {
-        reader.selfCheck(schemaMap, chunkGroupMetadataList, false);
+        if (reader.selfCheck(schemaMap, chunkGroupMetadataList, false)
+            != TsFileCheckStatus.COMPLETE_FILE) {
+          throw new QueryProcessException(
+              String.format(
+                  "Cannot load file %s because the file has crashed.", file.getAbsolutePath()));
+        }
         if (plan.getVerifyMetadata()) {
           loadNewTsFileVerifyMetadata(reader);
         }

diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -32,6 +32,7 @@
 import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
 import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType;
 import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
 import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
 import org.apache.iotdb.tsfile.read.common.Chunk;
 import org.apache.iotdb.tsfile.read.common.Path;
@@ -85,7 +86,10 @@ public TsFileSketchTool(String filename, String outFile) {
       // get metadata information
       tsFileMetaData = reader.readFileMetadata();
       allChunkGroupMetadata = new ArrayList<>();
-      reader.selfCheck(null, allChunkGroupMetadata, false);
+      if (reader.selfCheck(null, allChunkGroupMetadata, false) != TsFileCheckStatus.COMPLETE_FILE) {
+        throw new IOException(
+            String.format("Cannot load file %s because the file has crashed.", filename));
+      }
     } catch (IOException e) {
       e.printStackTrace();
     }

diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1257,15 +1257,16 @@ public long selfCheck(
     }
 
     tsFileInput.position(headerLength);
+    boolean isComplete = isComplete();
     if (fileSize == headerLength) {
       return headerLength;
-    } else if (isComplete()) {
+    } else if (isComplete) {
       loadMetadataSize();
       if (fastFinish) {
         return TsFileCheckStatus.COMPLETE_FILE;
       }
     }
-    // not a complete file, we will recover it...
+    // if not a complete file, we will recover it...
     long truncatedSize = headerLength;
     byte marker;
     List<long[]> timeBatch = new ArrayList<>();
@@ -1309,7 +1310,10 @@ public long selfCheck(
                 while (dataSize > 0) {
                   // a new Page
                   PageHeader pageHeader = this.readPageHeader(chunkHeader.getDataType(), true);
-                  chunkStatistics.mergeStatistics(pageHeader.getStatistics());
+                  if (pageHeader.getUncompressedSize() != 0) {
+                    // not empty page
+                    chunkStatistics.mergeStatistics(pageHeader.getStatistics());
+                  }
                   this.skipPageData(pageHeader);
                   dataSize -= pageHeader.getSerializedPageSize();
                   chunkHeader.increasePageNums(1);
@@ -1476,9 +1480,13 @@ public long selfCheck(
         // last chunk group Metadata
         chunkGroupMetadataList.add(new ChunkGroupMetadata(lastDeviceId, chunkMetadataList));
       }
-      truncatedSize = this.position() - 1;
+      if (isComplete) {
+        truncatedSize = TsFileCheckStatus.COMPLETE_FILE;
+      } else {
+        truncatedSize = this.position() - 1;
+      }
     } catch (Exception e) {
-      logger.info(
+      logger.warn(
           "TsFile {} self-check cannot proceed at position {} " + "recovered, because : {}",
           file,
           this.position(),

diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
@@ -636,8 +636,4 @@ public void close() throws IOException {
   public TsFileIOWriter getIOWriter() {
     return this.fileWriter;
   }
-
-  public void setIsUnseq(boolean unseq) {
-    this.isUnseq = unseq;
-  }
 }
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
@@ -20,19 +20,28 @@
 package org.apache.iotdb.tsfile.read;
 
 import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
 import org.apache.iotdb.tsfile.file.MetaMarker;
 import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
 import org.apache.iotdb.tsfile.file.header.ChunkHeader;
 import org.apache.iotdb.tsfile.file.header.PageHeader;
 import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
 import org.apache.iotdb.tsfile.utils.FileGenerator;
 import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.TsFileGeneratorUtils;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
 
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -125,4 +134,42 @@ public void testReadChunkMetadataInDevice() throws IOException {
     Assert.assertTrue(reader.readChunkMetadataInDevice("d3").isEmpty());
     reader.close();
   }
+
+  @Test
+  public void testReadEmptyPageInSelfCheck() throws IOException, WriteProcessException {
+    int oldMaxPagePointNum =
+        TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+    TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(10);
+    File testFile = new File(FILE_PATH);
+
+    // create tsfile with empty page
+    try (TsFileWriter tsFileWriter = new TsFileWriter(testFile)) {
+      // register aligned timeseries
+      List<MeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
+      alignedMeasurementSchemas.add(
+          new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN));
+      alignedMeasurementSchemas.add(
+          new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.PLAIN));
+      tsFileWriter.registerAlignedTimeseries(new Path("d1"), alignedMeasurementSchemas);
+
+      List<MeasurementSchema> writeMeasurementScheams = new ArrayList<>();
+      // only write s1
+      writeMeasurementScheams.add(alignedMeasurementSchemas.get(0));
+      TsFileGeneratorUtils.writeWithTsRecord(
+          tsFileWriter, "d1", writeMeasurementScheams, 25, 0, 0, true);
+
+      // write s1 and s2, fill 2 empty pages for s2
+      writeMeasurementScheams.add(alignedMeasurementSchemas.get(1));
+      TsFileGeneratorUtils.writeWithTsRecord(
+          tsFileWriter, "d1", writeMeasurementScheams, 10, 25, 0, true);
+    } finally {
+      TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldMaxPagePointNum);
+    }
+
+    // read tsfile with selfCheck method
+    TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH);
+    Assert.assertEquals(
+        TsFileCheckStatus.COMPLETE_FILE,
+        reader.selfCheck(new HashMap<>(), new ArrayList<>(), false));
+  }
 }