Added file overlap analysis tool

iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.bat

@@ -0,0 +1,62 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM     http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+
+@echo off
+echo ````````````````````````
+echo Starting Validating the TsFile
+echo ````````````````````````
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..\..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.storageengine.dataregion.compaction.tool.OverlapStatisticTool
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM -----------------------------------------------------------------------------
+@REM ***** CLASSPATH library setting *****
+@REM Ensure that any user defined CLASSPATH variables are not used on startup
+set CLASSPATH="%IOTDB_HOME%\lib\*"
+
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+@REM -----------------------------------------------------------------------------
+:okClasspath
+
+"%JAVA_HOME%\bin\java" -Xmx16G -cp "%CLASSPATH%" %MAIN_CLASS% %*
+
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM -----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL

iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Validating the TsFile
+echo ---------------------
+
+source "$(dirname "$0")/../../sbin/iotdb-common.sh"
+#get_iotdb_include and checkAllVariables is in iotdb-common.sh
+VARS=$(get_iotdb_include "$*")
+checkAllVariables
+export IOTDB_HOME="${IOTDB_HOME}/.."
+eval set -- "$VARS"
+
+if [ -n "$JAVA_HOME" ]; then
+    for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+        if [ -x "$java" ]; then
+            JAVA="$java"
+            break
+        fi
+    done
+else
+    JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+  CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.db.storageengine.dataregion.compaction.tool.OverlapStatisticTool
+
+"$JAVA" -Xmx16G -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/ITimeRange.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.tool;
+
+public interface ITimeRange {
+
+  // Add a time period to the current time range object. The increase process requires maintenance
+  // of the current TimeRange to facilitate efficient overlap check in the future
+  void addInterval(Interval interval);
+
+  // Determines whether the incoming time range overlaps with the current time range
+  boolean isOverlapped(Interval interval);
+}

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/Interval.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.tool;
+
+public class Interval {
+  private long start;
+  private long end;
+
+  public Interval(long start, long end) {
+    this.start = start;
+    this.end = end;
+    if (end < start) {
+      throw new IllegalArgumentException("end must greater than start");
+    }
+  }
+
+  public long getStart() {
+    return start;
+  }
+
+  public long getEnd() {
+    return end;
+  }
+
+  public void setStart(long start) {
+    this.start = start;
+  }
+
+  public void setEnd(long end) {
+    this.end = end;
+  }
+}

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/ListTimeRangeImpl.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.tool;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+public class ListTimeRangeImpl implements ITimeRange {
+
+  List<Interval> intervalList = new LinkedList<>();
+  // 0-10， 20-30， 50-70
+  // 25-60
+  // 0-10. 20-70
+
+  @Override
+  public void addInterval(Interval interval) {
+    List<Interval> mergedIntervals = new ArrayList<>();
+    int index = 0;
+
+    // 1. elements that do not overlap with the newly added element are placed directly in the
+    // result
+    while (index < intervalList.size() && intervalList.get(index).getEnd() < interval.getStart()) {
+      mergedIntervals.add(intervalList.get(index));
+      index++;
+    }
+
+    // 2. if the element overlaps with an existing element, start equals the minimum value of the
+    // overlap and end equals the maximum value of the overlap
+    while (index < intervalList.size() && intervalList.get(index).getStart() <= interval.getEnd()) {
+      interval.setStart(Math.min(intervalList.get(index).getStart(), interval.getStart()));
+      interval.setEnd(Math.max(intervalList.get(index).getEnd(), interval.getEnd()));
+      index++;
+    }
+    mergedIntervals.add(interval);
+
+    // 3. add the remaining elements to the result set
+    while (index < intervalList.size()) {
+      mergedIntervals.add(intervalList.get(index));
+      index++;
+    }
+
+    intervalList.clear();
+    intervalList.addAll(mergedIntervals);
+  }
+
+  public List<Interval> getIntervalList() {
+    return intervalList;
+  }
+
+  /**
+   * case 1: interval.getStart() <= currentInterval.getEnd()
+   *
+   * <p>currentInterval: [5,10], interval: [6,15],[1,7],[0,5],[10,15]
+   *
+   * <p>case 2: interval.getEnd() <= currentInterval.getEnd()
+   *
+   * <p>currentInterval: [5,10], interval:[1,9],[0,9],[1,10]
+   */
+  @Override
+  public boolean isOverlapped(Interval interval) {
+    for (Interval currentInterval : intervalList) {
+      boolean isOverlap =
+          interval.getStart() <= currentInterval.getEnd()
+              && interval.getEnd() >= currentInterval.getStart();
+      if (isOverlap) {
+        return true;
+      }
+    }
+    return false;
+  }
+}

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatistic.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.tool;
+
+import java.util.HashSet;
+
+public class OverlapStatistic {
+  long totalSequenceFile;
+  long totalSequenceFileSize;
+  long totalChunkGroupsInSequenceFile;
+  long totalChunksInSequenceFile;
+
+  HashSet<String> sequenceNumber = new HashSet<>();
+  long sequenceMinStartTime = Long.MAX_VALUE;
+  long sequenceMaxEndTime = Long.MIN_VALUE;
+
+  long totalUnsequenceFile;
+  long totalUnsequenceFileSize;
+  long totalChunkGroupsInUnSequenceFile;
+  long totalChunksInUnSequenceFile;
+  long unSequenceMinStartTime = Long.MAX_VALUE;
+  long unSequenceMaxEndTime = Long.MIN_VALUE;
+
+  long overlappedSequenceFiles;
+  long overlappedChunkGroupsInSequenceFile;
+  long overlappedChunksInSequenceFile;
+
+  public void merge(OverlapStatistic other) {
+    this.totalSequenceFile += other.totalSequenceFile;
+    this.totalSequenceFileSize += other.totalSequenceFileSize;
+    this.totalChunkGroupsInSequenceFile += other.totalChunkGroupsInSequenceFile;
+    this.totalChunksInSequenceFile += other.totalChunksInSequenceFile;
+    this.sequenceMinStartTime = Math.min(this.sequenceMinStartTime, other.sequenceMinStartTime);
+    this.sequenceMaxEndTime = Math.max(this.sequenceMaxEndTime, other.sequenceMaxEndTime);
+
+    this.totalUnsequenceFile += other.totalUnsequenceFile;
+    this.totalUnsequenceFileSize += other.totalUnsequenceFileSize;
+    this.totalChunkGroupsInUnSequenceFile += other.totalChunkGroupsInUnSequenceFile;
+    this.totalChunksInUnSequenceFile += other.totalChunksInUnSequenceFile;
+    this.unSequenceMinStartTime =
+        Math.min(this.unSequenceMinStartTime, other.unSequenceMinStartTime);
+    this.unSequenceMaxEndTime = Math.max(this.unSequenceMaxEndTime, other.unSequenceMaxEndTime);
+
+    this.overlappedSequenceFiles += other.overlappedSequenceFiles;
+    this.overlappedChunkGroupsInSequenceFile += other.overlappedChunkGroupsInSequenceFile;
+    this.overlappedChunksInSequenceFile += other.overlappedChunksInSequenceFile;
+  }
+
+  public void mergeSingleSequenceFileTaskResult(SequenceFileTaskSummary summary) {
+    if (summary.equals(new SequenceFileTaskSummary())) {
+      return;
+    }
+    if (summary.overlapChunkGroup > 0) {
+      this.overlappedSequenceFiles += 1;
+    }
+    this.overlappedChunkGroupsInSequenceFile += summary.overlapChunkGroup;
+    this.totalChunkGroupsInSequenceFile += summary.totalChunkGroups;
+    this.overlappedChunksInSequenceFile += summary.overlapChunk;
+    this.totalChunksInSequenceFile += summary.totalChunks;
+    this.totalSequenceFile += 1;
+    this.totalSequenceFileSize += summary.fileSize;
+    this.sequenceMinStartTime = Math.min(this.sequenceMinStartTime, summary.minStartTime);
+    this.sequenceMaxEndTime = Math.max(this.sequenceMaxEndTime, summary.maxEndTime);
+  }
+
+  public void mergeUnSeqSpaceStatistics(UnseqSpaceStatistics statistics) {
+    this.totalUnsequenceFile += statistics.unsequenceFileNum;
+    this.totalUnsequenceFileSize += statistics.unsequenceFileSize;
+    this.totalChunksInUnSequenceFile += statistics.unsequenceChunkNum;
+    this.totalChunkGroupsInUnSequenceFile += statistics.unsequenceChunkGroupNum;
+    this.unSequenceMinStartTime = Math.min(this.unSequenceMinStartTime, statistics.minStartTime);
+    this.unSequenceMaxEndTime = Math.max(this.unSequenceMaxEndTime, statistics.maxEndTime);
+  }
+}