Skip to content
Permalink
Browse files
[IOTDB-3656] load-tsfile-tool rename to rewrite-tsfile-tool (#6635)
  • Loading branch information
yschengzi committed Jul 19, 2022
1 parent 198f38a commit 116b4b2393e6350ec009cb27f3a58a44fec051ca
Show file tree
Hide file tree
Showing 17 changed files with 45 additions and 42 deletions.
@@ -19,18 +19,18 @@
-->

# IoTDB Load-TsFile Tool
# IoTDB Rewrite-TsFile Tool

## Introduction

The Load-TsFile tool is used to write the data in TsFile to the running IoTDB.
The Rewrite-TsFile tool is used to write the data in TsFile to the running IoTDB.

## How to use

Run load-tsfile.bat if you are in a Windows environment, or load-tsfile.sh if you are on Linux or Unix.
Run rewrite-tsfile.bat if you are in a Windows environment, or rewrite-tsfile.sh if you are on Linux or Unix.

```bash
./load-tsfile.bat -f filePath [-h host] [-help] [-p port] [-pw password] -u user
./rewrite-tsfile.bat -f filePath [-h host] [-help] [-p port] [-pw password] -u user
-f File or Dictionary to be loaded.
-h Host Name (optional, default 127.0.0.1)
-help Display help information(optional)
@@ -43,17 +43,17 @@ Run load-tsfile.bat if you are in a Windows environment, or load-tsfile.sh if yo

Assuming that an IoTDB instance is running on server 192.168.0.101:6667, you want to load all TsFile files from the locally saved TsFile backup folder D:\IoTDB\data into this IoTDB instance.

First move to the folder where load-tsfile.bat is located, open the command line, and execute
First move to the folder where rewrite-tsfile.bat is located, open the command line, and execute

```bash
./load-tsfile.bat -f "D:\IoTDB\data" -h 192.168.0.101 -p 6667 -u root -pw root
./load-rewrite.bat -f "D:\IoTDB\data" -h 192.168.0.101 -p 6667 -u root -pw root
```

After waiting for the script execution to complete, you can check that the data in the IoTDB instance has been loaded correctly.

## Q&A

- Cannot find or load the main class TsFileLoaderTool
- Cannot find or load the main class RewriteTsFileTool
- It may be because the environment variable $IOTDB_HOME is not set, please set the environment variable and try again
- Missing require argument: f or Missing require argument: u
- The input command is missing the -f field (file or folder path to be loaded) or the -u field (user name), please add it and re-execute
@@ -19,18 +19,18 @@
-->

# IoTDB Load-TsFile Tool
# IoTDB Rewrite-TsFile Tool

## 工具介绍

Load-TsFile工具用于将TsFile中的数据写入正在运行的IoTDB中。
Rewrite-TsFile工具用于将TsFile中的数据写入正在运行的IoTDB中。

## 使用方法

若您在Windows环境中,请运行load-tsfile.bat,若为Linux或Unix,请运行load-tsfile.sh
若您在Windows环境中,请运行rewrite-tsfile.bat,若为Linux或Unix,请运行rewrite-tsfile.sh

```bash
./load-tsfile.bat -f filePath [-h host] [-help] [-p port] [-pw password] -u user
./rewrite-tsfile.bat -f filePath [-h host] [-help] [-p port] [-pw password] -u user
-f 待加载的文件或文件夹路径,必要字段
-h IoTDB的Host地址,可选,默认127.0.0.1
-help 输出帮助菜单,可选
@@ -43,17 +43,17 @@ Load-TsFile工具用于将TsFile中的数据写入正在运行的IoTDB中。

假定服务器192.168.0.101:6667上运行一个IoTDB实例,想从将本地保存的TsFile备份文件夹D:\IoTDB\data中的所有的TsFile文件都加载进此IoTDB实例。

首先移动到load-tsfile.bat所在文件夹中,打开命令行,然后执行
首先移动到rewrite-tsfile.bat所在文件夹中,打开命令行,然后执行

```bash
./load-tsfile.bat -f "D:\IoTDB\data" -h 192.168.0.101 -p 6667 -u root -pw root
./rewrite-tsfile.bat -f "D:\IoTDB\data" -h 192.168.0.101 -p 6667 -u root -pw root
```

等待脚本执行完成之后,可以检查IoTDB实例中数据已经被正确加载

## 常见问题

- 找不到或无法加载主类TsFileLoaderTool
- 找不到或无法加载主类RewriteTsFileTool
- 可能是由于未设置环境变量$IOTDB_HOME,请设置环境变量之后重试
- Missing require argument: f或Missing require argument: u
- 输入命令缺少待-f字段(加载文件或文件夹路径),或者缺少-u字段(用户名),请添加之后重新执行
@@ -74,7 +74,7 @@
</dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
<artifactId>load-tsfile-tool</artifactId>
<artifactId>rewrite-tsfile-tool</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
@@ -63,7 +63,7 @@
</dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
<artifactId>load-tsfile-tool</artifactId>
<artifactId>rewrite-tsfile-tool</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
@@ -18,7 +18,7 @@
*/
package org.apache.iotdb.db.integration;

import org.apache.iotdb.TsFileLoaderTool;
import org.apache.iotdb.RewriteTsFileTool;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -38,7 +38,7 @@
import java.sql.ResultSet;
import java.sql.Statement;

public class IoTDBTsFileLoaderToolIT {
public class IoTDBRewriteTsFileToolIT {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();

private String tmpDir;
@@ -137,7 +137,7 @@ public void testLoadTsFile() {
prepareTsFiles();

String[] args = {"-h", "127.0.0.1", "-p", "6667", "-u", "root", "-pw", "root", "-f", tmpDir};
TsFileLoaderTool.main(args);
RewriteTsFileTool.main(args);
try (Connection connection =
DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667", "root", "root");
Statement statement = connection.createStatement()) {
@@ -119,7 +119,7 @@
<module>library-udf</module>
<module>schema-engine-rocksdb</module>
<module>udf-api</module>
<module>load-tsfile-tool</module>
<module>rewrite-tsfile-tool</module>
</modules>
<!-- Properties Management -->
<properties>
@@ -27,7 +27,7 @@
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>load-tsfile-tool</artifactId>
<artifactId>rewrite-tsfile-tool</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.iotdb</groupId>
@@ -61,7 +61,7 @@
</goals>
<configuration>
<descriptors>
<descriptor>src/assembly/loadTsFileTool.xml</descriptor>
<descriptor>src/assembly/rewriteTsFile.xml</descriptor>
</descriptors>
<appendAssemblyId>false</appendAssemblyId>
<archive>
@@ -28,7 +28,7 @@ pushd %~dp0..
if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
popd

if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.TsFileLoaderTool
if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.RewriteTsFileTool
if NOT DEFINED JAVA_HOME goto :err

@REM -----------------------------------------------------------------------------
@@ -42,7 +42,7 @@ for f in ${IOTDB_HOME}/lib/*.jar; do
CLASSPATH=${CLASSPATH}":"$f
done

MAIN_CLASS=org.apache.iotdb.TsFileLoaderTool
MAIN_CLASS=org.apache.iotdb.RewriteTsFileTool

"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
exit $?
@@ -73,7 +73,7 @@
import java.util.Map;
import java.util.Set;

public class TsFileLoaderTool {
public class RewriteTsFileTool {
private static final int MAX_TABLET_LENGTH = 1024 * 64;

private static String host = "localhost";
@@ -126,7 +126,8 @@ public static void parseArgs(String[] args) {
}

private static void priHelp(Options options) {
new HelpFormatter().printHelp("./load-tsfile.sh(load-tsfile.bat if Windows)", options, true);
new HelpFormatter()
.printHelp("./rewrite-tsfile.sh(rewrite-tsfile.bat if Windows)", options, true);
}

private static String getArgOrDefault(
@@ -163,7 +163,7 @@
import org.apache.iotdb.db.sync.sender.pipe.Pipe;
import org.apache.iotdb.db.sync.sender.pipe.PipeSink;
import org.apache.iotdb.db.sync.sender.service.SenderService;
import org.apache.iotdb.db.tools.TsFileRewriteTool;
import org.apache.iotdb.db.tools.TsFileSplitByPartitionTool;
import org.apache.iotdb.db.utils.FileLoaderUtils;
import org.apache.iotdb.db.utils.TypeInferenceUtils;
import org.apache.iotdb.db.utils.UpgradeUtils;
@@ -1516,7 +1516,7 @@ private void loadFile(File file, OperateFilePlan plan) throws QueryProcessExcept
logger.info(
"try to split the tsFile={} du to it spans multi partitions",
tsFileResource.getTsFile().getPath());
TsFileRewriteTool.rewriteTsFile(tsFileResource, splitResources);
TsFileSplitByPartitionTool.rewriteTsFile(tsFileResource, splitResources);
tsFileResource.writeLock();
tsFileResource.removeModFile();
tsFileResource.writeUnlock();
@@ -23,7 +23,7 @@
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.exception.sync.PipeDataLoadException;
import org.apache.iotdb.db.exception.sync.PipeDataLoadUnbearableException;
import org.apache.iotdb.db.tools.TsFileRewriteTool;
import org.apache.iotdb.db.tools.TsFileSplitByPartitionTool;
import org.apache.iotdb.db.utils.FileLoaderUtils;

import java.io.File;
@@ -47,7 +47,7 @@ public void load() throws PipeDataLoadException {
FileLoaderUtils.loadOrGenerateResource(tsFileResource);
List<TsFileResource> splitResources = new ArrayList();
if (tsFileResource.isSpanMultiTimePartitions()) {
TsFileRewriteTool.rewriteTsFile(tsFileResource, splitResources);
TsFileSplitByPartitionTool.rewriteTsFile(tsFileResource, splitResources);
tsFileResource.writeLock();
tsFileResource.removeModFile();
tsFileResource.writeUnlock();
@@ -67,9 +67,9 @@
import java.util.Map;
import java.util.Map.Entry;

public class TsFileRewriteTool implements AutoCloseable {
public class TsFileSplitByPartitionTool implements AutoCloseable {

private static final Logger logger = LoggerFactory.getLogger(TsFileRewriteTool.class);
private static final Logger logger = LoggerFactory.getLogger(TsFileSplitByPartitionTool.class);

protected TsFileSequenceReader reader;
protected File oldTsFile;
@@ -98,7 +98,7 @@ public class TsFileRewriteTool implements AutoCloseable {
*
* @throws IOException If some I/O error occurs
*/
public TsFileRewriteTool(TsFileResource resourceToBeRewritten) throws IOException {
public TsFileSplitByPartitionTool(TsFileResource resourceToBeRewritten) throws IOException {
oldTsFileResource = resourceToBeRewritten;
oldTsFile = resourceToBeRewritten.getTsFile();
String file = oldTsFile.getAbsolutePath();
@@ -110,7 +110,7 @@ public TsFileRewriteTool(TsFileResource resourceToBeRewritten) throws IOExceptio
}
}

public TsFileRewriteTool(TsFileResource resourceToBeRewritten, boolean needReaderForV2)
public TsFileSplitByPartitionTool(TsFileResource resourceToBeRewritten, boolean needReaderForV2)
throws IOException {
oldTsFileResource = resourceToBeRewritten;
oldTsFile = resourceToBeRewritten.getTsFile();
@@ -136,7 +136,8 @@ public TsFileRewriteTool(TsFileResource resourceToBeRewritten, boolean needReade
public static void rewriteTsFile(
TsFileResource resourceToBeRewritten, List<TsFileResource> rewrittenResources)
throws IOException, WriteProcessException, IllegalPathException {
try (TsFileRewriteTool rewriteTool = new TsFileRewriteTool(resourceToBeRewritten)) {
try (TsFileSplitByPartitionTool rewriteTool =
new TsFileSplitByPartitionTool(resourceToBeRewritten)) {
rewriteTool.parseAndRewriteFile(rewrittenResources);
}
}
@@ -24,7 +24,7 @@
import org.apache.iotdb.db.engine.settle.SettleLog.SettleCheckStatus;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.tools.TsFileRewriteTool;
import org.apache.iotdb.db.tools.TsFileSplitByPartitionTool;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
@@ -212,7 +212,8 @@ public void settleOneTsFileAndMod(
if (!resourceToBeSettled.getModFile().exists()) {
return;
}
try (TsFileRewriteTool tsFileRewriteTool = new TsFileRewriteTool(resourceToBeSettled)) {
try (TsFileSplitByPartitionTool tsFileRewriteTool =
new TsFileSplitByPartitionTool(resourceToBeSettled)) {
tsFileRewriteTool.parseAndRewriteFile(settledResources);
}
}
@@ -21,7 +21,7 @@
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.tools.TsFileRewriteTool;
import org.apache.iotdb.db.tools.TsFileSplitByPartitionTool;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
@@ -47,7 +47,7 @@
import java.util.List;
import java.util.Map;

public class TsFileOnlineUpgradeTool extends TsFileRewriteTool {
public class TsFileOnlineUpgradeTool extends TsFileSplitByPartitionTool {

private static final Logger logger = LoggerFactory.getLogger(TsFileOnlineUpgradeTool.class);

@@ -31,7 +31,7 @@
import org.apache.iotdb.db.qp.Planner;
import org.apache.iotdb.db.qp.executor.IPlanExecutor;
import org.apache.iotdb.db.qp.executor.PlanExecutor;
import org.apache.iotdb.db.tools.TsFileRewriteTool;
import org.apache.iotdb.db.tools.TsFileSplitByPartitionTool;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
@@ -236,7 +236,7 @@ private void splitFileAndQueryCheck(HashMap<String, List<String>> deviceSensorsM
TsFileResource tsFileResource = new TsFileResource(tsFile);
List<TsFileResource> splitResource = new ArrayList<>();
try {
TsFileRewriteTool.rewriteTsFile(tsFileResource, splitResource);
TsFileSplitByPartitionTool.rewriteTsFile(tsFileResource, splitResource);
} catch (IOException | WriteProcessException | IllegalPathException e) {
Assert.fail(e.getMessage());
}
@@ -416,7 +416,7 @@ private void splitTwoPagesFileAndQueryCheck(String device, String sensor) {
TsFileResource tsFileResource = new TsFileResource(tsFile);
List<TsFileResource> splitResource = new ArrayList<>();
try {
TsFileRewriteTool.rewriteTsFile(tsFileResource, splitResource);
TsFileSplitByPartitionTool.rewriteTsFile(tsFileResource, splitResource);
} catch (IOException | WriteProcessException | IllegalPathException e) {
Assert.fail(e.getMessage());
}

0 comments on commit 116b4b2

Please sign in to comment.