From a01763047867b70f076c4102d2167294b1ac6aef Mon Sep 17 00:00:00 2001 From: "luke.miao" <282583553@qq.com> Date: Tue, 30 Apr 2024 13:27:36 +0800 Subject: [PATCH 1/5] Fixed backup path detection and optimized output log file names --- .../src/assembly/resources/tools/backup.bat | 8 +++- .../src/assembly/resources/tools/backup.sh | 7 ++-- .../apache/iotdb/tool/IoTDBDataBackTool.java | 39 +++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.bat b/iotdb-client/cli/src/assembly/resources/tools/backup.bat index 4037f5d3ff37a..4ef8316bcd53e 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.bat @@ -91,15 +91,19 @@ echo ------------------------------------------ echo Starting IoTDB Client Data Back Script echo ------------------------------------------ -set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*" +set CLASSPATH="%IOTDB_HOME%\lib\*" if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.IoTDBDataBackTool set logsDir="%IOTDB_HOME%\logs" if not exist "%logsDir%" ( mkdir "%logsDir%" ) +for /F "tokens=2 delims==" %%G in ('wmic OS Get localdatetime /value') do set "logDate=%%G" +set "logDate=%logDate:~0,8%" -start /B "" cmd /C "%JAVA_HOME%\bin\java -DIOTDB_HOME=!IOTDB_HOME! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %* > %logsDir%\iotdb-data-back.log 2>&1" +set "logFilename=log-iotdb-data-back-%logDate%.log" + +start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DIOTDB_HOME=!IOTDB_HOME! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > "%logsDir%\%logFilename%" 2>&1" exit /b :checkIfIOTDBProcess diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.sh b/iotdb-client/cli/src/assembly/resources/tools/backup.sh index 8cc66daf6c69c..4a9dd615bcc97 100755 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.sh @@ -111,8 +111,6 @@ check_running_process() { done if [ ${#iotdb_listening_ports[@]} -gt 0 ]; then echo " Please stop IoTDB" >&2 - echo " Exit..." >&2 - echo "" exit 1 fi } @@ -131,4 +129,7 @@ if [ ! -d "$logs_dir" ]; then mkdir "$logs_dir" fi -nohup "$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" > ${logs_dir}/iotdb-data-back.log 2>&1 & +logDate=$(date "+%Y%m%d") +logFilename="log-iotdb-data-back-${logDate}.log" + +nohup "$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" > "${logs_dir}/${logFilename}" 2>&1 & diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java index 1938831205014..948431cea5609 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java @@ -188,6 +188,8 @@ public static void main(String[] args) { countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties); countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); countNodeBack(targetDirString.toString(), copyMap); + + for (Map.Entry entry : copyMap.entrySet()) { countFiles(entry.getKey()); } @@ -225,6 +227,7 @@ public static void main(String[] args) { for (Map.Entry entry : copyMap.entrySet()) { countFiles(entry.getKey()); } + isDirectoryInsideOrSame(copyMap); ioTDBDataBack(copyMap, dnDataDirsMap); propertiesFileUpdate( targetDirString.toString() @@ -242,6 +245,8 @@ public static void main(String[] args) { for (Map.Entry entry : dnDataDirsMap.entrySet()) { countFiles(entry.getKey()); } + isDirectoryInsideOrSame(copyMap); + isDirectoryInsideOrSame(dnDataDirsMap); ioTDBDataBack(copyMap, dnDataDirsMap); propertiesFileUpdate( targetDirString.toString() @@ -260,6 +265,10 @@ public static void main(String[] args) { for (Map.Entry entry : dnDataDirsMap.entrySet()) { countFiles(entry.getKey()); } + + isDirectoryInsideOrSame(copyMap); + isDirectoryInsideOrSame(dnDataDirsMap); + ioTDBDataBack(copyMap, dnDataDirsMap); propertiesFileUpdate( targetDirString.toString() @@ -281,6 +290,36 @@ public static void main(String[] args) { delFile(filename); } + private static void isDirectoryInsideOrSame(Map copyMap) { + for (Map.Entry sourceEntry : copyMap.entrySet()) { + for (Map.Entry targetEntry : copyMap.entrySet()) { + File file = new File(sourceEntry.getKey()); + if(file.exists()){ + Path targetPath=Paths.get(targetEntry.getValue()); + Path parentPath=Paths.get(sourceEntry.getKey()); + Path normalizedTargetPath = targetPath.normalize(); + Path normalizedParentPath = parentPath.normalize(); + if(normalizedTargetPath.startsWith(normalizedParentPath) || normalizedTargetPath.equals(normalizedParentPath)){ + if(targetDirParam.length()>0 && targetDataDirParam.length()>0 && targetWalDirParam.length()>0){ + LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{},{},{}", targetDirParam,targetDataDirParam,targetWalDirParam); + System.exit(0); + }else if(targetDirParam.length()>0&& targetDataDirParam.length()>0){ + LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{},{}", targetDirParam,targetDataDirParam); + System.exit(0); + }else if(targetDirParam.length()>0 && targetWalDirParam.length()>0){ + LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{},{}", targetDirParam,targetWalDirParam); + System.exit(0); + }else if(targetDirParam.length()>0){ + LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{}", targetDirParam); + System.exit(0); + } + + } + } + } + } + } + private static void ioTDBDataBack( Map copyMap, Map dnDataDirsMap) { From 85bf69cb4751250a94bb6a31881bd14446fa5192 Mon Sep 17 00:00:00 2001 From: "luke.miao" <282583553@qq.com> Date: Tue, 30 Apr 2024 14:26:50 +0800 Subject: [PATCH 2/5] spotless --- .../apache/iotdb/tool/IoTDBDataBackTool.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java index 948431cea5609..640e1efe0ab04 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java @@ -189,7 +189,6 @@ public static void main(String[] args) { countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); countNodeBack(targetDirString.toString(), copyMap); - for (Map.Entry entry : copyMap.entrySet()) { countFiles(entry.getKey()); } @@ -290,30 +289,44 @@ public static void main(String[] args) { delFile(filename); } - private static void isDirectoryInsideOrSame(Map copyMap) { + private static void isDirectoryInsideOrSame(Map copyMap) { for (Map.Entry sourceEntry : copyMap.entrySet()) { for (Map.Entry targetEntry : copyMap.entrySet()) { File file = new File(sourceEntry.getKey()); - if(file.exists()){ - Path targetPath=Paths.get(targetEntry.getValue()); - Path parentPath=Paths.get(sourceEntry.getKey()); + if (file.exists()) { + Path targetPath = Paths.get(targetEntry.getValue()); + Path parentPath = Paths.get(sourceEntry.getKey()); Path normalizedTargetPath = targetPath.normalize(); Path normalizedParentPath = parentPath.normalize(); - if(normalizedTargetPath.startsWith(normalizedParentPath) || normalizedTargetPath.equals(normalizedParentPath)){ - if(targetDirParam.length()>0 && targetDataDirParam.length()>0 && targetWalDirParam.length()>0){ - LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{},{},{}", targetDirParam,targetDataDirParam,targetWalDirParam); + if (normalizedTargetPath.startsWith(normalizedParentPath) + || normalizedTargetPath.equals(normalizedParentPath)) { + if (targetDirParam.length() > 0 + && targetDataDirParam.length() > 0 + && targetWalDirParam.length() > 0) { + LOGGER.error( + "The directory to be backed up cannot be in the source directory, please check:{},{},{}", + targetDirParam, + targetDataDirParam, + targetWalDirParam); System.exit(0); - }else if(targetDirParam.length()>0&& targetDataDirParam.length()>0){ - LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{},{}", targetDirParam,targetDataDirParam); + } else if (targetDirParam.length() > 0 && targetDataDirParam.length() > 0) { + LOGGER.error( + "The directory to be backed up cannot be in the source directory, please check:{},{}", + targetDirParam, + targetDataDirParam); System.exit(0); - }else if(targetDirParam.length()>0 && targetWalDirParam.length()>0){ - LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{},{}", targetDirParam,targetWalDirParam); + } else if (targetDirParam.length() > 0 && targetWalDirParam.length() > 0) { + LOGGER.error( + "The directory to be backed up cannot be in the source directory, please check:{},{}", + targetDirParam, + targetWalDirParam); System.exit(0); - }else if(targetDirParam.length()>0){ - LOGGER.error("The directory to be backed up cannot be in the source directory, please check:{}", targetDirParam); + } else if (targetDirParam.length() > 0) { + LOGGER.error( + "The directory to be backed up cannot be in the source directory, please check:{}", + targetDirParam); System.exit(0); } - } } } From 0227dd39576010f016691a4c11423985e6b87080 Mon Sep 17 00:00:00 2001 From: "luke.miao" <282583553@qq.com> Date: Mon, 6 May 2024 15:04:01 +0800 Subject: [PATCH 3/5] Fixed issues with non-standard naming of backup data directories and incorrect data directory backup --- .../src/assembly/resources/tools/backup.bat | 129 ++++++++++++++ .../src/assembly/resources/tools/backup.sh | 135 +++++++++++++++ .../apache/iotdb/tool/IoTDBDataBackTool.java | 157 +++++++++++++----- 3 files changed, 383 insertions(+), 38 deletions(-) create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.bat create mode 100755 iotdb-client/cli/src/assembly/resources/tools/backup.sh diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.bat b/iotdb-client/cli/src/assembly/resources/tools/backup.bat new file mode 100644 index 0000000000000..4ef8316bcd53e --- /dev/null +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.bat @@ -0,0 +1,129 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off +setlocal enabledelayedexpansion + +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +popd + +if NOT DEFINED JAVA_HOME goto :err + +set JAVA_OPTS=-ea^ + -DIOTDB_HOME="%IOTDB_HOME%" + +SET IOTDB_CONF=%IOTDB_HOME%\conf +IF EXIST "%IOTDB_CONF%\datanode-env.bat" ( + CALL "%IOTDB_CONF%\datanode-env.bat" > nul 2>&1 +) ELSE ( + echo Can't find datanode-env.bat +) + +IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( + for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" + "%IOTDB_CONF%\iotdb-datanode.properties"') do ( + set dn_rpc_port=%%i + ) +) ELSE ( + set dn_rpc_port=6667 +) + +IF EXIST "%IOTDB_CONF%\iotdb-confignode.properties" ( + for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" + "%IOTDB_CONF%\iotdb-confignode.properties"') do ( + set cn_internal_port=%%i + ) +) ELSE ( + set cn_internal_port=10710 +) + +set "local_iotdb_occupied_ports=" +set "operation_dirs=" +set dn_rpc_port_occupied=0 +set cn_internal_port_occupied=0 + +for /f "tokens=1,3,7 delims=: " %%i in ('netstat /ano') do ( + if %%i==TCP ( + if %%j==%dn_rpc_port% ( + if !dn_rpc_port_occupied!==0 ( + set spid=%%k + call :checkIfIOTDBProcess !spid! is_iotdb + if !is_iotdb!==1 ( + set local_iotdb_occupied_ports=%dn_rpc_port% !local_iotdb_occupied_ports! + ) + ) + + ) else if %%j==%cn_internal_port% ( + if !cn_internal_port_occupied!==0 ( + set spid=%%k + call :checkIfIOTDBProcess !spid! is_iotdb + if !is_iotdb!==1 ( + set local_iotdb_occupied_ports=%cn_internal_port% !local_iotdb_occupied_ports! + ) + ) + ) + ) +) + +if defined local_iotdb_occupied_ports ( + goto :checkFail +) +echo ------------------------------------------ +echo Starting IoTDB Client Data Back Script +echo ------------------------------------------ + +set CLASSPATH="%IOTDB_HOME%\lib\*" +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.IoTDBDataBackTool + +set logsDir="%IOTDB_HOME%\logs" +if not exist "%logsDir%" ( + mkdir "%logsDir%" +) +for /F "tokens=2 delims==" %%G in ('wmic OS Get localdatetime /value') do set "logDate=%%G" +set "logDate=%logDate:~0,8%" + +set "logFilename=log-iotdb-data-back-%logDate%.log" + +start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DIOTDB_HOME=!IOTDB_HOME! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > "%logsDir%\%logFilename%" 2>&1" +exit /b + +:checkIfIOTDBProcess +setlocal + +set "pid_to_check=%~1" +set "is_iotdb=0" + +for /f "usebackq tokens=*" %%i in (`wmic process where "ProcessId=%pid_to_check%" get CommandLine /format:list ^| findstr /c:"CommandLine="`) do ( + set command_line=%%i +) +echo %command_line% | findstr /i /c:"iotdb" >nul && set is_iotdb=1 +endlocal & set "is_iotdb=%is_iotdb%" +exit /b + +:err +echo JAVA_HOME environment variable must be set! +set ret_code=1 +exit /b + +:checkFail +echo Please stop IoTDB +exit /b \ No newline at end of file diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.sh b/iotdb-client/cli/src/assembly/resources/tools/backup.sh new file mode 100755 index 0000000000000..4a9dd615bcc97 --- /dev/null +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +echo ------------------------------------------ +echo Starting IoTDB Client Data Back Script +echo ------------------------------------------ + + +if [ -z "${IOTDB_INCLUDE}" ]; then + #do nothing + : +elif [ -r "$IOTDB_INCLUDE" ]; then + . "$IOTDB_INCLUDE" +fi + +if [ -z "${IOTDB_HOME}" ]; then + export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +if [ -z $JAVA ] ; then + echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr + exit 1; +fi + +datanodeclassname=org.apache.iotdb.db.service.DataNode + +confignodeclassname=org.apache.iotdb.db.service.ConfigNode + +check_tool_env() { + if ! type lsof > /dev/null 2>&1 ; then + echo "" + echo " Warning: No tool 'lsof', Please install it." + echo " Note: Some checking function need 'lsof'." + echo "" + return 1 + else + return 0 + fi +} + +get_properties_value() { + local file_name=$1 + local property_name=$2 + local default_value=$3 + local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${IOTDB_HOME}/conf/${file_name}.properties") + if [ -z "$property_value" ]; then + property_value="$default_value" + fi + echo "$property_value" +} +iotdb_listening_ports=() +check_running_process() { + DATANODE="iotdb-datanode" + CONFIGNODE="iotdb-confignode" + dn_rpc_port=$(get_properties_value $DATANODE "dn_rpc_port" "6667") + cn_internal_port=$(get_properties_value $CONFIGNODE "cn_internal_port" "10710") + local_ports+=("$dn_rpc_port") + local_ports+=("$cn_internal_port") + for port in "${local_ports[@]}"; do + # Check if lsof command is available + if command -v lsof >/dev/null 2>&1; then + listening=$(lsof -i :$port -sTCP:LISTEN -P -n | grep LISTEN) + if [ -n "$listening" ]; then + process_command=$(echo "$listening" | awk '{print $2}') + iotdb_check=$(ps -p "$process_command" -o args= | grep "iotdb") + if [ -n "$iotdb_check" ]; then + iotdb_listening_ports+=("$port ") + fi + fi + elif command -v netstat >/dev/null 2>&1; then + listening=$(netstat -tln | awk '{print $4}' | grep ":$port$") + if [ -n "$listening" ]; then + process_command=$(echo "$listening" | awk '{print $2}') + iotdb_check=$(ps -p "$process_command" -o args= | grep "iotdb") + if [ -n "$iotdb_check" ]; then + iotdb_listening_ports+=("$port ") + fi + fi + else + echo "Error: Unable to detect port occupation. Please install 'lsof' or 'netstat' command." + exit 1 + fi + done + if [ ${#iotdb_listening_ports[@]} -gt 0 ]; then + echo " Please stop IoTDB" >&2 + exit 1 + fi +} + +check_running_process + +for f in ${IOTDB_HOME}/lib/*.jar; do + CLASSPATH=${CLASSPATH}":"$f +done + +MAIN_CLASS=org.apache.iotdb.tool.IoTDBDataBackTool + +logs_dir="${IOTDB_HOME}/logs" + +if [ ! -d "$logs_dir" ]; then + mkdir "$logs_dir" +fi + +logDate=$(date "+%Y%m%d") +logFilename="log-iotdb-data-back-${logDate}.log" + +nohup "$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" > "${logs_dir}/${logFilename}" 2>&1 & diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java index 640e1efe0ab04..41b77edeb42a2 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java @@ -66,7 +66,7 @@ public class IoTDBDataBackTool { static AtomicInteger fileCount = new AtomicInteger(0); static AtomicInteger targetFileCount = new AtomicInteger(0); static AtomicInteger processFileCount = new AtomicInteger(0); - static final String filename = "data.txt"; + static final String filename = "backup.log"; private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDataTool.class); @@ -185,32 +185,69 @@ public static void main(String[] args) { System.exit(0); } - countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties); - countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); - countNodeBack(targetDirString.toString(), copyMap); + if (nodeTypeParam.equalsIgnoreCase("confignode")) { + countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties); + countNodeBack(targetDirString.toString(), copyMap); + for (Map.Entry entry : copyMap.entrySet()) { + countFiles(entry.getKey()); + } - for (Map.Entry entry : copyMap.entrySet()) { - countFiles(entry.getKey()); - } - for (Map.Entry entry : dnDataDirsMap.entrySet()) { - countFiles(entry.getKey()); + ioTDBDataBack(copyMap, dnDataDirsMap); + propertiesFileUpdate( + targetDirString.toString() + + File.separatorChar + + "conf" + + File.separatorChar + + CONFIG_NODE_CONF_NAME, + cnMapProperties); + } else if (nodeTypeParam.equalsIgnoreCase("datanode")) { + countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); + countNodeBack(targetDirString.toString(), copyMap); + checkQuickMode(dnDataDirsMap); + for (Map.Entry entry : copyMap.entrySet()) { + countFiles(entry.getKey()); + } + for (Map.Entry entry : dnDataDirsMap.entrySet()) { + countFiles(entry.getKey()); + } + + ioTDBDataBack(copyMap, dnDataDirsMap); + propertiesFileUpdate( + targetDirString.toString() + + File.separatorChar + + "conf" + + File.separatorChar + + DATA_NODE_CONF_NAME, + dnMapProperties); + } else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) { + countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties); + countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); + countNodeBack(targetDirString.toString(), copyMap); + checkQuickMode(dnDataDirsMap); + for (Map.Entry entry : copyMap.entrySet()) { + countFiles(entry.getKey()); + } + for (Map.Entry entry : dnDataDirsMap.entrySet()) { + countFiles(entry.getKey()); + } + + ioTDBDataBack(copyMap, dnDataDirsMap); + propertiesFileUpdate( + targetDirString.toString() + + File.separatorChar + + "conf" + + File.separatorChar + + CONFIG_NODE_CONF_NAME, + cnMapProperties); + propertiesFileUpdate( + targetDirString.toString() + + File.separatorChar + + "conf" + + File.separatorChar + + DATA_NODE_CONF_NAME, + dnMapProperties); } - ioTDBDataBack(copyMap, dnDataDirsMap); - propertiesFileUpdate( - targetDirString.toString() - + File.separatorChar - + "conf" - + File.separatorChar - + CONFIG_NODE_CONF_NAME, - cnMapProperties); - propertiesFileUpdate( - targetDirString.toString() - + File.separatorChar - + "conf" - + File.separatorChar - + DATA_NODE_CONF_NAME, - dnMapProperties); } else { if (targetDirParam != null && targetDirParam.length() > 0) { targetDirString.append(targetDirParam); @@ -238,6 +275,7 @@ public static void main(String[] args) { } else if (nodeTypeParam.equalsIgnoreCase("datanode")) { countNodeBack(targetDirString.toString(), copyMap); countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); + for (Map.Entry entry : copyMap.entrySet()) { countFiles(entry.getKey()); } @@ -289,6 +327,16 @@ public static void main(String[] args) { delFile(filename); } + private static void checkQuickMode(Map dnDataDirsMap) { + for (Map.Entry entry : dnDataDirsMap.entrySet()) { + File backupDir = new File(entry.getValue()); + if (backupDir.exists()) { + LOGGER.error("The backup folder already exists:{}", entry.getValue()); + System.exit(0); + } + } + } + private static void isDirectoryInsideOrSame(Map copyMap) { for (Map.Entry sourceEntry : copyMap.entrySet()) { for (Map.Entry targetEntry : copyMap.entrySet()) { @@ -456,7 +504,16 @@ private static void countDataNodeFile( } String targetDnDataDirs = ""; if (targetDataDirParam.isEmpty()) { - targetDnDataDirs = sourceDnDataDirsCoverTargetDnDataDirsHandler(dnDataDirs); + targetDataDirParam = + targetDirParam + + File.separatorChar + + "data" + + File.separatorChar + + "datanode" + + File.separatorChar + + "data"; + targetDnDataDirs = getCreateDnDataPathString(dnDataDirs, targetDataDirParam, "data"); + // targetDnDataDirs = sourceDnDataDirsCoverTargetDnDataDirsHandler(dnDataDirs); } else { targetDnDataDirs = getCreateDnDataPathString(dnDataDirs, targetDataDirParam, "data"); } @@ -558,7 +615,7 @@ public static String sourceDnDataDirsCoverTargetDnDataDirsHandler(String dnDirs) StringBuilder subTargetDataDir = new StringBuilder(); for (int i = 0; i < sourcePathList.length; i++) { String path = sourcePathList[i]; - path = path + "_back"; + path = path + "_backup"; if (i == sourcePathList.length - 1) { subTargetDataDir.append(path); } else { @@ -579,7 +636,16 @@ public static String sourceWalCoverTargetWalDirsHandler(String dnDirs, String ta StringBuilder subTargetDataDir = new StringBuilder(); for (int i = 0; i < sourcePathList.length; i++) { if (i == sourcePathList.length - 1) { - subTargetDataDir.append(targetDirs).append(File.separatorChar).append("wal").append(i + 1); + if (sourcePathList.length == 1) { + subTargetDataDir.append(targetDirs); + } else { + subTargetDataDir + .append(targetDirs) + .append(File.separatorChar) + .append("wal") + .append(i + 1); + } + } else { subTargetDataDir .append(targetDirs) @@ -611,7 +677,9 @@ public static String getCreateDnDataPathString( String[] sourcePathArray = sourcePathsList[i].split(","); for (int j = 0; j < sourcePathArray.length; j++) { String newPath = targetPathsList[0]; - if (sourcePathsList.length == 1 && targetPathsList.length == 1) { + if (sourcePathsList.length == 1 + && targetPathsList.length == 1 + && sourcePathArray.length == 1) { newPath = newPath + File.separatorChar + dirType; } else { newPath = newPath + File.separatorChar + dirType + num; @@ -635,7 +703,9 @@ public static String getCreateDnDataPathString( if (targetPathArray.length == 1) { for (int j = 0; j < sourcePathArray.length; j++) { String newPath = ""; - if (sourcePathsList.length == 1 && targetPathsList.length == 1) { + if (sourcePathsList.length == 1 + && targetPathsList.length == 1 + && sourcePathArray.length == 1) { newPath = targetPathArray[0] + File.separatorChar + dirType; } else { newPath = targetPathArray[0] + File.separatorChar + dirType + num; @@ -682,7 +752,9 @@ public static Map getCreatePathMapping( String[] sourcePathArray = sourcePathsList[i].split(","); for (int j = 0; j < sourcePathArray.length; j++) { String newPath = targetPathsList[0]; - if (sourcePathsList.length == 1 && targetPathsList.length == 1) { + if (sourcePathsList.length == 1 + && targetPathsList.length == 1 + && sourcePathArray.length == 1) { newPath = newPath + File.separatorChar + dirType; } else { newPath = newPath + File.separatorChar + dirType + num; @@ -701,7 +773,9 @@ public static Map getCreatePathMapping( if (targetPathArray.length == 1) { for (int j = 0; j < sourcePathArray.length; j++) { String newPath = ""; - if (sourcePathsList.length == 1 && targetPathsList.length == 1) { + if (sourcePathsList.length == 1 + && targetPathsList.length == 1 + && sourcePathArray.length == 1) { newPath = targetPathArray[0] + File.separatorChar + dirType; } else { newPath = targetPathArray[0] + File.separatorChar + dirType + num; @@ -823,7 +897,10 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) if (processFileCount.get() > targetFileCount.get()) { writeFileData(filename, processFileCount.get()); LOGGER.info( - "total file number:" + fileCount + ",backup file number:" + processFileCount); + "total file number:" + + fileCount + + ",verify the number of files:" + + targetFileCount); } else { writeFileData(filename, targetFileCount.get()); LOGGER.info( @@ -877,12 +954,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) try { Files.createLink(targetFile, file); } catch (UnsupportedOperationException | IOException e) { - LOGGER.error("link file error {}", sourceDirectory, e); + LOGGER.error("link file error {}", e); try { Files.copy(file, targetFile, StandardCopyOption.REPLACE_EXISTING); } catch (IOException ex) { targetFileCount.decrementAndGet(); - LOGGER.error("copy file error {}", sourceDirectory, ex); + LOGGER.error("copy file error {}", ex); } } } @@ -890,8 +967,10 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) if (processFileCount.get() > targetFileCount.get()) { writeFileData(filename, processFileCount.get()); LOGGER.info( - "total file number:" + fileCount + ",backup file number:" + processFileCount); - + "total file number:" + + fileCount + + ",verify the number of files:" + + targetFileCount); } else { writeFileData(filename, targetFileCount.get()); LOGGER.info( @@ -994,10 +1073,10 @@ private static void countFilesRecursively(File file, AtomicInteger fileCount) { } public static int readFileData(String filename) { - Path filePath = Paths.get(filename); + filename = sourcePath + File.separatorChar + "logs" + File.separatorChar + filename; createFile(filename); - try { + Path filePath = Paths.get(filename); List lines = Files.readAllLines(filePath, StandardCharsets.UTF_8); if (!lines.isEmpty()) { return Integer.parseInt(lines.get(0)); @@ -1009,6 +1088,7 @@ public static int readFileData(String filename) { } public static void delFile(String filename) { + filename = sourcePath + File.separatorChar + "logs" + File.separatorChar + filename; File file = new File(filename); if (file.exists()) { file.delete(); @@ -1016,6 +1096,7 @@ public static void delFile(String filename) { } public static void writeFileData(String filename, int data) { + filename = sourcePath + File.separatorChar + "logs" + File.separatorChar + filename; Path filePath = Paths.get(filename); try { Files.write(filePath, Integer.toString(data).getBytes(StandardCharsets.UTF_8)); From 662b504cb5e2b831812808326ca5f72dd5a633c4 Mon Sep 17 00:00:00 2001 From: "luke.miao" <282583553@qq.com> Date: Tue, 7 May 2024 14:56:24 +0800 Subject: [PATCH 4/5] add logback --- .../assembly/resources/conf/logback-cli.xml | 19 ++++++++++++++++++- .../src/assembly/resources/tools/backup.bat | 8 +++----- .../src/assembly/resources/tools/backup.sh | 7 +++---- .../apache/iotdb/tool/IoTDBDataBackTool.java | 14 +++++++++++--- 4 files changed, 35 insertions(+), 13 deletions(-) mode change 100755 => 100644 iotdb-client/cli/src/assembly/resources/tools/backup.sh diff --git a/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml b/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml index 1f80786a9c585..4f35ee0407256 100644 --- a/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml +++ b/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml @@ -26,7 +26,24 @@ ERROR - + + UTF-8 + ${IOTDB_HOME}/logs/log-iotdb-data-back.log + + ${IOTDB_HOME}/logs/log-iotdb-data-back-%d{yyyyMMdd}.log.gz + 30 + + true + + %d [%t] %-5p %C{25}:%L - %m %n + utf-8 + + + INFO + + + + diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.bat b/iotdb-client/cli/src/assembly/resources/tools/backup.bat index 4ef8316bcd53e..315fd44e53cda 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.bat @@ -98,12 +98,10 @@ set logsDir="%IOTDB_HOME%\logs" if not exist "%logsDir%" ( mkdir "%logsDir%" ) -for /F "tokens=2 delims==" %%G in ('wmic OS Get localdatetime /value') do set "logDate=%%G" -set "logDate=%logDate:~0,8%" -set "logFilename=log-iotdb-data-back-%logDate%.log" - -start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DIOTDB_HOME=!IOTDB_HOME! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > "%logsDir%\%logFilename%" 2>&1" +set IOTDB_CLI_CONF=%IOTDB_HOME%\conf +set "iotdb_cli_params=-Dlogback.configurationFile=!IOTDB_CLI_CONF!\logback-cli.xml" +start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DIOTDB_HOME=!IOTDB_HOME! !iotdb_cli_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > nul 2>&1" exit /b :checkIfIOTDBProcess diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.sh b/iotdb-client/cli/src/assembly/resources/tools/backup.sh old mode 100755 new mode 100644 index 4a9dd615bcc97..6d605812825c3 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.sh @@ -129,7 +129,6 @@ if [ ! -d "$logs_dir" ]; then mkdir "$logs_dir" fi -logDate=$(date "+%Y%m%d") -logFilename="log-iotdb-data-back-${logDate}.log" - -nohup "$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" > "${logs_dir}/${logFilename}" 2>&1 & +IOTDB_CLI_CONF=${IOTDB_HOME}/conf +iotdb_cli_params="-Dlogback.configurationFile=${IOTDB_CLI_CONF}/logback-cli.xml" +exec nohup "$JAVA" -DIOTDB_HOME=${IOTDB_HOME} $iotdb_cli_params -cp "$CLASSPATH" "$MAIN_CLASS" "$@" 2>&1 > /dev/null <&- & diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java index 41b77edeb42a2..7d3315d1f67d6 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java @@ -159,6 +159,7 @@ public static boolean vaildParam(String dnDataDirs, String dnWalDirs) { } public static void main(String[] args) { + System.setProperty("IOTDB_HOME", System.getenv("IOTDB_HOME")); argsParse(args); File sourceDir = new File(sourcePath); Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME); @@ -178,14 +179,15 @@ public static void main(String[] args) { .append(File.separatorChar) .append("iotdb_backup"); File targetDir = new File(targetDirString.toString()); - if (!targetDir.exists()) { - targetDir.mkdirs(); - } else { + if (targetDir.exists()) { LOGGER.error("The backup folder already exists:{}", targetDirString); System.exit(0); } if (nodeTypeParam.equalsIgnoreCase("confignode")) { + if (!targetDir.exists()) { + targetDir.mkdirs(); + } countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties); countNodeBack(targetDirString.toString(), copyMap); for (Map.Entry entry : copyMap.entrySet()) { @@ -204,6 +206,9 @@ public static void main(String[] args) { countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); countNodeBack(targetDirString.toString(), copyMap); checkQuickMode(dnDataDirsMap); + if (!targetDir.exists()) { + targetDir.mkdirs(); + } for (Map.Entry entry : copyMap.entrySet()) { countFiles(entry.getKey()); } @@ -224,6 +229,9 @@ public static void main(String[] args) { countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); countNodeBack(targetDirString.toString(), copyMap); checkQuickMode(dnDataDirsMap); + if (!targetDir.exists()) { + targetDir.mkdirs(); + } for (Map.Entry entry : copyMap.entrySet()) { countFiles(entry.getKey()); } From ae4cdc5dfbefe0c7127a1a44dac158b1c7d411ac Mon Sep 17 00:00:00 2001 From: "luke.miao" <282583553@qq.com> Date: Tue, 7 May 2024 15:59:49 +0800 Subject: [PATCH 5/5] add logback --- distribution/src/assembly/all.xml | 5 +++++ distribution/src/assembly/cli.xml | 5 +++++ iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/distribution/src/assembly/all.xml b/distribution/src/assembly/all.xml index dd95f2a4e73e7..5768f197563fc 100644 --- a/distribution/src/assembly/all.xml +++ b/distribution/src/assembly/all.xml @@ -88,6 +88,11 @@ ${maven.multiModuleProjectDirectory}/iotdb-client/cli/src/assembly/resources/tools 0755 + + conf + ${maven.multiModuleProjectDirectory}/iotdb-client/cli/src/assembly/resources/conf + 0755 + diff --git a/distribution/src/assembly/cli.xml b/distribution/src/assembly/cli.xml index 701b1bfa50fe1..0258de92a24ca 100644 --- a/distribution/src/assembly/cli.xml +++ b/distribution/src/assembly/cli.xml @@ -42,6 +42,11 @@ ${maven.multiModuleProjectDirectory}/iotdb-client/cli/src/assembly/resources/sbin 0755 + + conf + ${maven.multiModuleProjectDirectory}/iotdb-client/cli/src/assembly/resources/conf + 0755 + tools ${maven.multiModuleProjectDirectory}/iotdb-client/cli/src/assembly/resources/tools diff --git a/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml b/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml index 4f35ee0407256..78fa22572f077 100644 --- a/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml +++ b/iotdb-client/cli/src/assembly/resources/conf/logback-cli.xml @@ -28,9 +28,9 @@ UTF-8 - ${IOTDB_HOME}/logs/log-iotdb-data-back.log + ${IOTDB_HOME}/logs/log_backup.log - ${IOTDB_HOME}/logs/log-iotdb-data-back-%d{yyyyMMdd}.log.gz + ${IOTDB_HOME}/logs/log-backup-%d{yyyyMMdd}.log.gz 30 true