[IOTDB-141]Develop memory estimate tool

iotdb/iotdb/bin/memory-tool.bat

@@ -0,0 +1,67 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM     http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+set IOTDB_CONF=%IOTDB_HOME%\conf
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM -----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -Dlogback.configurationFile="%IOTDB_CONF%\logback-tool.xml"^
+ -DIOTDB_HOME=%IOTDB_HOME%
+
+@REM ***** CLASSPATH library setting *****
+@REM Ensure that any user defined CLASSPATH variables are not used on startup
+set CLASSPATH="%IOTDB_HOME%\lib"
+
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+@REM -----------------------------------------------------------------------------
+:okClasspath
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
+
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM -----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL

iotdb/iotdb/bin/memory-tool.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${IOTDB_HOME}" ]; then
+  export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+IOTDB_CONF=${IOTDB_HOME}/conf
+# IOTDB_LOGS=${IOTDB_HOME}/logs
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+  CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
+
+if [ -n "$JAVA_HOME" ]; then
+    for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+        if [ -x "$java" ]; then
+            JAVA="$java"
+            break
+        fi
+    done
+else
+    JAVA=java
+fi
+
+iotdb_parms="-Dlogback.configurationFile=${IOTDB_CONF}/logback-tool.xml"
+
+exec "$JAVA" $iotdb_parms -cp "$CLASSPATH" "$MAIN_CLASS" "$@"

iotdb/iotdb/bin/start-server.bat

@@ -78,7 +78,7 @@ set JAVA_OPTS=-ea^
 @REM Ensure that any user defined CLASSPATH variables are not used on startup
 set CLASSPATH="%IOTDB_HOME%\lib"
 
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
 for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
 set CLASSPATH=%CLASSPATH%;iotdb.IoTDB
 goto okClasspath
@@ -87,7 +87,7 @@ goto okClasspath
 set CLASSPATH=%CLASSPATH%;%1
 goto :eof
 
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
 :okClasspath
 
 rem echo CLASSPATH: %CLASSPATH%

iotdb/iotdb/bin/start-sync-client.bat

@@ -44,7 +44,7 @@ set JAVA_OPTS=-ea^
 @REM Ensure that any user defined CLASSPATH variables are not used on startup
 set CLASSPATH="%IOTDB_HOME%\lib"
 
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
 for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
 set CLASSPATH=%CLASSPATH%;SyncClient
 goto okClasspath
@@ -53,10 +53,10 @@ goto okClasspath
 set CLASSPATH=%CLASSPATH%;%1
 goto :eof
 
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
 :okClasspath
 
-rem echo CLASSPATH: %CLASSPATH%
+@rem echo CLASSPATH: %CLASSPATH%
 
 "%JAVA_HOME%\bin\java" %JAVA_OPTS% %IOTDB_HEAP_OPTS% -cp %CLASSPATH% %IOTDB_JMX_OPTS% %MAIN_CLASS%
 goto finally

iotdb/iotdb/conf/logback-tool.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<configuration/>

iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java

@@ -26,7 +26,6 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;

iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java

@@ -211,7 +211,7 @@ public class IoTDBConfig {
   /**
    * The memory used for stat performance.
    */
-  private int performance_stat_memory_in_kb = 20;
+  private int performanceStatMemoryInKB = 20;
   /**
    * whether use chunkBufferPool.
    */
@@ -565,12 +565,12 @@ public void setPerformanceStatDisplayInterval(long performanceStatDisplayInterva
     this.performanceStatDisplayInterval = performanceStatDisplayInterval;
   }
 
-  public int getPerformance_stat_memory_in_kb() {
-    return performance_stat_memory_in_kb;
+  public int getPerformanceStatMemoryInKB() {
+    return performanceStatMemoryInKB;
   }
 
-  public void setPerformance_stat_memory_in_kb(int performance_stat_memory_in_kb) {
-    this.performance_stat_memory_in_kb = performance_stat_memory_in_kb;
+  public void setPerformanceStatMemoryInKB(int performanceStatMemoryInKB) {
+    this.performanceStatMemoryInKB = performanceStatMemoryInKB;
   }
 
   public long getMemtableSizeThreshold() {

iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java

@@ -226,9 +226,9 @@ private void loadProps() {
       conf.setPerformanceStatDisplayInterval(Long
           .parseLong(properties.getProperty("performance_stat_display_interval",
               Long.toString(conf.getPerformanceStatDisplayInterval())).trim()));
-      conf.setPerformance_stat_memory_in_kb(Integer
+      conf.setPerformanceStatMemoryInKB(Integer
           .parseInt(properties.getProperty("performance_stat_memory_in_kb",
-              Integer.toString(conf.getPerformance_stat_memory_in_kb())).trim()));
+              Integer.toString(conf.getPerformanceStatMemoryInKB())).trim()));
     } catch (IOException e) {
       logger.warn("Cannot load config file because, use default configuration", e);
     } catch (Exception e) {

iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java

@@ -68,11 +68,7 @@ public class CompressionRatio {
   private CompressionRatio() {
     directory = new File(
         FilePathUtils.regularizePath(CONFIG.getSystemDir()) + COMPRESSION_RATIO_DIR);
-    try {
-      restore();
-    } catch (IOException e) {
-      LOGGER.error("Can not restore CompressionRatio", e);
-    }
+    restore();
   }
 
   /**
@@ -123,8 +119,10 @@ private void checkDirectoryExist() throws IOException {
   /**
    * Restore compression ratio statistics from disk when system restart
    */
-  void restore() throws IOException {
-    checkDirectoryExist();
+  void restore() {
+    if (!directory.exists()) {
+      return;
+    }
     File[] ratioFiles = directory.listFiles((dir, name) -> name.startsWith(FILE_PREFIX));
     if (ratioFiles != null && ratioFiles.length > 0) {
       long maxTimes = 0;

iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java

@@ -20,7 +20,6 @@
 
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.StorageEngine;
 import org.apache.iotdb.db.exception.ConfigAdjusterException;
 import org.apache.iotdb.db.metadata.MManager;
 import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
@@ -37,10 +36,11 @@
  *
  * 1. maxMemTableNum. This parameter represents the size of the MemTable available in the MemTable
  * pool, which is closely related to the number of storage groups. When adding or deleting a storage
- * group, the parameter also adds or deletes two MemTables. The reason why adding or deleting two
+ * group, the parameter also adds or deletes four MemTables. The reason why adding or deleting four
  * MemTables is that when the system is running stably, the speed of the flush operation is faster
  * than that of data writing, so one is used for the Flush process and the other is used for data
- * writing. Otherwise, the system should limit the speed of data writing to maintain stability.
+ * writing. Otherwise, the system should limit the speed of data writing to maintain stability. And
+ * two for sequence data, two for unsequence data.
  *
  * 2. memtableSize. This parameter determines the threshold value for the MemTable in memory to be
  * flushed into disk. When the system load increases, the parameter should be set smaller so that
@@ -75,16 +75,10 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
 
   // static parameter section
 
-  /**
-   * When the size of the adjusted MemTable decreases more than this parameter, trigger the global
-   * flush operation and flush all MemTable that meets the flush condition to disk.
-   */
-  private static final double FLUSH_THRESHOLD = 0.2;
-
   /**
    * Maximum amount of memory allocated for write process.
    */
-  private static final long ALLOCATE_MEMORY_FOR_WRITE = CONFIG.getAllocateMemoryForWrite();
+  private static long allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite();
 
   /**
    * Metadata size of per timeseries, the default value is 2KB.
@@ -168,7 +162,7 @@ private int calcMemTableSize() {
     // when unit is byte, it's likely to cause Long type overflow.
     // so when b is larger than Integer.MAC_VALUE use the unit KB.
     double a = ratio * maxMemTableNum;
-    double b = (ALLOCATE_MEMORY_FOR_WRITE - staticMemory) * ratio;
+    double b = (allocateMemoryForWrite - staticMemory) * ratio;
     int magnification = b > Integer.MAX_VALUE ? 1024 : 1;
     b /= magnification;
     double c = (double) CONFIG.getTsFileSizeThreshold() * maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE
@@ -187,7 +181,7 @@ private int calcMemTableSize() {
    * @return Tsfile byte threshold
    */
   private long calcTsFileSize(long memTableSize) {
-    return (long) ((ALLOCATE_MEMORY_FOR_WRITE - maxMemTableNum * memTableSize - staticMemory) * CompressionRatio
+    return (long) ((allocateMemoryForWrite - maxMemTableNum * memTableSize - staticMemory) * CompressionRatio
             .getInstance().getRatio()
             * memTableSize / (maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE * MManager.getInstance()
             .getMaximalSeriesNumberAmongStorageGroups()));
@@ -253,6 +247,7 @@ public void reset() {
     totalTimeseries = 0;
     staticMemory = 0;
     maxMemTableNum = MEM_TABLE_AVERAGE_QUEUE_LEN;
+    allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite();
     initialized = false;
   }
 

iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java

@@ -120,7 +120,7 @@ private Measurement() {
     IoTDBConfig tdbConfig = IoTDBDescriptor.getInstance().getConfig();
     isEnableStat = tdbConfig.isEnablePerformanceStat();
     displayIntervalInMs = tdbConfig.getPerformanceStatDisplayInterval();
-    int memoryInKb = tdbConfig.getPerformance_stat_memory_in_kb();
+    int memoryInKb = tdbConfig.getPerformanceStatMemoryInKB();
 
     queueSize = memoryInKb * 1000 / Operation.values().length / 8;
     operationLatenciesQueue = new ConcurrentCircularArray[Operation.values().length];

iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java

@@ -180,54 +180,6 @@ public void delete(Deletion deletion) {
     this.modifications.add(deletion);
   }
 
-  /**
-   * If chunk contains data with timestamp less than 'timestamp', create a copy and delete all those
-   * data. Otherwise return null.
-   *
-   * @param chunk the source chunk.
-   * @param timestamp the upper-bound of deletion time.
-   * @return A reduced copy of chunk if chunk contains data with timestamp less than 'timestamp', of
-   * null.
-   */
-  private IWritableMemChunk filterChunk(IWritableMemChunk chunk, long timestamp) {
-
-    if (!chunk.isEmpty() && chunk.getMinTime() <= timestamp) {
-      //TODO we can avoid sorting data here by scanning data once.
-      List<TimeValuePair> timeValuePairs = chunk.getSortedTimeValuePairList();
-      TSDataType dataType = chunk.getType();
-      IWritableMemChunk newChunk = genMemSeries(dataType);
-      for (TimeValuePair pair : timeValuePairs) {
-        if (pair.getTimestamp() > timestamp) {
-          switch (dataType) {
-            case BOOLEAN:
-              newChunk.putBoolean(pair.getTimestamp(), pair.getValue().getBoolean());
-              break;
-            case DOUBLE:
-              newChunk.putDouble(pair.getTimestamp(), pair.getValue().getDouble());
-              break;
-            case INT64:
-              newChunk.putLong(pair.getTimestamp(), pair.getValue().getLong());
-              break;
-            case INT32:
-              newChunk.putInt(pair.getTimestamp(), pair.getValue().getInt());
-              break;
-            case FLOAT:
-              newChunk.putFloat(pair.getTimestamp(), pair.getValue().getFloat());
-              break;
-            case TEXT:
-              newChunk.putBinary(pair.getTimestamp(), pair.getValue().getBinary());
-              break;
-            default:
-                throw new UnsupportedOperationException("Unknown datatype: " + dataType);
-          }
-        }
-      }
-      TVListAllocator.getInstance().release(dataType, chunk.getTVList());
-      return newChunk;
-    }
-    return null;
-  }
-
   public void setVersion(long version) {
     this.version = version;
   }