[IOTDB-1635] settle TsFiles and mods (#4024)

.asf.yaml

@@ -30,4 +30,4 @@ github:
     - tsdb
   features:
     wiki: true
-    issues: true
+    issues: true

.mvn/wrapper/MavenWrapperDownloader.java

@@ -112,4 +112,4 @@ protected PasswordAuthentication getPasswordAuthentication() {
     }
   }
 
-}
+}

antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4

@@ -45,6 +45,7 @@ statement
     | DROP INDEX indexName=ID ON prefixPath #dropIndex //not support yet
     | MERGE #merge
     | FLUSH prefixPath? (COMMA prefixPath)* (booleanClause)?#flush
+    | SETTLE pathOrString #settle
     | FULL MERGE #fullMerge
     | CLEAR CACHE #clearcache
     | CREATE USER userName=ID password= stringLiteral#createUser
@@ -1068,6 +1069,10 @@ FLUSH
     : F L U S H
     ;
 
+SETTLE
+    : S E T T L E
+    ;
+
 TASK
     : T A S K
     ;
@@ -1193,7 +1198,7 @@ PLA
    ;
 
 LZ4
-   : L Z '4' 
+   : L Z '4'
    ;
 
 LATEST
@@ -1423,6 +1428,11 @@ stringLiteral
    | DOUBLE_QUOTE_STRING_LITERAL
    ;
 
+pathOrString
+   : prefixPath
+   | stringLiteral
+   ;
+
 INT : [0-9]+;
 
 EXPONENT : INT ('e'|'E') ('+'|'-')? INT ;

jenkins.pom

@@ -64,4 +64,4 @@
         </profile>
     </profiles>
 
-</project>
+</project>

server/src/assembly/resources/tools/tsfileToolSet/settle.bat

@@ -0,0 +1,62 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM     http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+
+@echo off
+echo ````````````````````````
+echo Starting Settling the TsFile
+echo ````````````````````````
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..\..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM -----------------------------------------------------------------------------
+@REM ***** CLASSPATH library setting *****
+@REM Ensure that any user defined CLASSPATH variables are not used on startup
+set CLASSPATH="%IOTDB_HOME%\lib\*"
+
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+@REM -----------------------------------------------------------------------------
+:okClasspath
+
+"%JAVA_HOME%\bin\java" -cp "%CLASSPATH%" %MAIN_CLASS% %*
+
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM -----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL

server/src/assembly/resources/tools/tsfileToolSet/settle.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Settling the TsFile
+echo ---------------------
+
+if [ -z "${IOTDB_HOME}" ]; then
+  export IOTDB_HOME="$(cd "`dirname "$0"`"/../..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+    for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+        if [ -x "$java" ]; then
+            JAVA="$java"
+            break
+        fi
+    done
+else
+    JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+  CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool
+
+"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?

server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java

@@ -535,6 +535,9 @@ public class IoTDBConfig {
   /** How many threads will be set up to perform upgrade tasks. */
   private int upgradeThreadNum = 1;
 
+  /** How many threads will be set up to perform settle tasks. */
+  private int settleThreadNum = 1;
+
   /** How many threads will be set up to perform main merge tasks. */
   private int mergeThreadNum = 1;
 
@@ -2055,6 +2058,10 @@ public int getUpgradeThreadNum() {
     return upgradeThreadNum;
   }
 
+  public int getSettleThreadNum() {
+    return settleThreadNum;
+  }
+
   void setUpgradeThreadNum(int upgradeThreadNum) {
     this.upgradeThreadNum = upgradeThreadNum;
   }

server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java

@@ -797,6 +797,31 @@ public void upgradeAll() throws StorageEngineException {
     }
   }
 
+  public void getResourcesToBeSettled(
+      PartialPath sgPath,
+      List<TsFileResource> seqResourcesToBeSettled,
+      List<TsFileResource> unseqResourcesToBeSettled,
+      List<String> tsFilePaths)
+      throws StorageEngineException {
+    VirtualStorageGroupManager vsg = processorMap.get(sgPath);
+    if (vsg == null) {
+      throw new StorageEngineException(
+          "The Storage Group " + sgPath.toString() + " is not existed.");
+    }
+    if (!vsg.getIsSettling().compareAndSet(false, true)) {
+      throw new StorageEngineException(
+          "Storage Group " + sgPath.getFullPath() + " is already being settled now.");
+    }
+    vsg.getResourcesToBeSettled(seqResourcesToBeSettled, unseqResourcesToBeSettled, tsFilePaths);
+  }
+
+  public void setSettling(PartialPath sgPath, boolean isSettling) {
+    if (processorMap.get(sgPath) == null) {
+      return;
+    }
+    processorMap.get(sgPath).setSettling(isSettling);
+  }
+
   /**
    * merge all storage groups.
    *

server/src/main/java/org/apache/iotdb/db/engine/settle/SettleLog.java

@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.settle;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class SettleLog {
+  private static final Logger logger = LoggerFactory.getLogger(SettleLog.class);
+  public static final String COMMA_SEPERATOR = ",";
+  private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+  private static final String SETTLE_DIR = "settle";
+  private static final String SETTLE_LOG_NAME = "settle.txt";
+  private static BufferedWriter settleLogWriter;
+  private static File settleLogPath = // the path of upgrade log is "data/system/settle/settle.txt"
+      SystemFileFactory.INSTANCE.getFile(
+          SystemFileFactory.INSTANCE.getFile(config.getSystemDir(), SETTLE_DIR), SETTLE_LOG_NAME);
+
+  private static final ReadWriteLock settleLogFileLock = new ReentrantReadWriteLock();
+
+  public static boolean createSettleLog() {
+    try {
+      if (!settleLogPath.getParentFile().exists()) {
+        settleLogPath.getParentFile().mkdirs();
+      }
+      settleLogPath.createNewFile();
+      settleLogWriter = new BufferedWriter(new FileWriter(getSettleLogPath(), true));
+      return true;
+    } catch (IOException e) {
+      logger.error("meet error when creating settle log, file path:{}", settleLogPath, e);
+      return false;
+    }
+  }
+
+  public static boolean writeSettleLog(String content) {
+    settleLogFileLock.writeLock().lock();
+    try {
+      settleLogWriter.write(content);
+      settleLogWriter.newLine();
+      settleLogWriter.flush();
+      return true;
+    } catch (IOException e) {
+      logger.error("write settle log file failed, the log file:{}", getSettleLogPath(), e);
+      return false;
+    } finally {
+      settleLogFileLock.writeLock().unlock();
+    }
+  }
+
+  public static void closeLogWriter() {
+    try {
+      if (settleLogWriter != null) {
+        settleLogWriter.close();
+      }
+    } catch (IOException e) {
+      logger.error("close upgrade log file failed, the log file:{}", getSettleLogPath(), e);
+    }
+  }
+
+  public static String getSettleLogPath() { // "data/system/settle/settle.txt"
+    return settleLogPath.getAbsolutePath();
+  }
+
+  public static void setSettleLogPath(File settleLogPath) {
+    SettleLog.settleLogPath = settleLogPath;
+  }
+
+  public enum SettleCheckStatus {
+    BEGIN_SETTLE_FILE(1),
+    AFTER_SETTLE_FILE(2),
+    SETTLE_SUCCESS(3);
+
+    private final int checkStatus;
+
+    SettleCheckStatus(int checkStatus) {
+      this.checkStatus = checkStatus;
+    }
+
+    public int getCheckStatus() {
+      return checkStatus;
+    }
+
+    @Override
+    public String toString() {
+      return String.valueOf(checkStatus);
+    }
+  }
+}