Skip to content

[To rel/0.12][IOTDB-842] Better Export/Import-CSV Tool #4045

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HTHou merged 2 commits into from
Sep 27, 2021
Merged

[To rel/0.12][IOTDB-842] Better Export/Import-CSV Tool #4045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
85b16b1
modify the user guide.
Sep 22, 2021
f1d9fc7
support new CSV tools in rel/0.12
Sep 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion checkstyle.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,4 +227,4 @@
</module>
<module name="CommentsIndentation"/>
</module>
</module>
</module>
5 changes: 5 additions & 0 deletions cli/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.9.0</version>
</dependency>
</dependencies>
<build>
<plugins>
Expand Down
14 changes: 12 additions & 2 deletions cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.iotdb.jdbc.IoTDBJDBCResultSet;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.service.rpc.thrift.ServerProperties;
import org.apache.iotdb.tool.ImportCsv;
Expand Down Expand Up @@ -478,8 +479,17 @@ private static void importCmd(String specialCmd, String cmd, IoTDBConnection con
return;
}
println(cmd.split(" ")[1]);
ImportCsv.importCsvFromFile(
host, port, username, password, cmd.split(" ")[1], connection.getTimeZone());
try {
ImportCsv.importFromTargetPath(
host,
Integer.valueOf(port),
username,
password,
cmd.split(" ")[1],
connection.getTimeZone());
} catch (IoTDBConnectionException e) {
e.printStackTrace();
}
}

private static void executeQuery(IoTDBConnection connection, String cmd) {
Expand Down
96 changes: 70 additions & 26 deletions cli/src/main/java/org/apache/iotdb/tool/AbstractCsvTool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,18 @@
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.Session;

import jline.console.ConsoleReader;
import jline.internal.Nullable;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.QuoteMode;

import java.io.IOException;
import java.io.PrintWriter;
import java.time.ZoneId;
import java.util.List;

public abstract class AbstractCsvTool {

Expand All @@ -55,33 +60,44 @@ public abstract class AbstractCsvTool {
protected static final int MAX_HELP_CONSOLE_WIDTH = 92;
protected static final String[] TIME_FORMAT =
new String[] {"default", "long", "number", "timestamp"};
protected static final String[] STRING_TIME_FORMAT =
public static final String[] STRING_TIME_FORMAT =
new String[] {
"yyyy-MM-dd'T'HH:mm:ss.SSSZ",
"yyyy/MM/dd HH:mm:ss.SSS",
"yyyy-MM-dd HH:mm:ss.SSSX",
"yyyy/MM/dd HH:mm:ss.SSSX",
"yyyy.MM.dd HH:mm:ss.SSSX",
"yyyy-MM-dd HH:mm:ssX",
"yyyy/MM/dd HH:mm:ssX",
"yyyy.MM.dd HH:mm:ssX",
"yyyy-MM-dd HH:mm:ss.SSSz",
"yyyy/MM/dd HH:mm:ss.SSSz",
"yyyy.MM.dd HH:mm:ss.SSSz",
"yyyy-MM-dd HH:mm:ssz",
"yyyy/MM/dd HH:mm:ssz",
"yyyy.MM.dd HH:mm:ssz",
"yyyy-MM-dd HH:mm:ss.SSS",
"yyyy/MM/dd HH:mm:ss.SSS",
"yyyy.MM.dd HH:mm:ss.SSS",
"yyyy/MM/dd'T'HH:mm:ss.SSS",
"yyyy-MM-dd'T'HH:mm:ss.SSS",
"yyyy-MM-dd'T'HH:mm:ss.SSS",
"yyyy.MM.dd'T'HH:mm:ss.SSS",
"yyyy-MM-dd HH:mm:ss.SSSZZ",
"yyyy/MM/dd HH:mm:ss.SSSZZ",
"yyyy.MM.dd HH:mm:ss.SSSZZ",
"yyyy-MM-dd'T'HH:mm:ss.SSSZZ",
"yyyy/MM/dd'T'HH:mm:ss.SSSZZ",
"yyyy-MM-dd HH:mm:ss",
"yyyy/MM/dd HH:mm:ss",
"yyyy.MM.dd HH:mm:ss",
"yyyy-MM-dd'T'HH:mm:ss.SSSX",
"yyyy/MM/dd'T'HH:mm:ss.SSSX",
"yyyy.MM.dd'T'HH:mm:ss.SSSX",
"yyyy-MM-dd'T'HH:mm:ssX",
"yyyy/MM/dd'T'HH:mm:ssX",
"yyyy.MM.dd'T'HH:mm:ssX",
"yyyy-MM-dd'T'HH:mm:ss.SSSz",
"yyyy/MM/dd'T'HH:mm:ss.SSSz",
"yyyy.MM.dd'T'HH:mm:ss.SSSz",
"yyyy-MM-dd'T'HH:mm:ssz",
"yyyy/MM/dd'T'HH:mm:ssz",
"yyyy.MM.dd'T'HH:mm:ssz",
"yyyy-MM-dd'T'HH:mm:ss.SSS",
"yyyy/MM/dd'T'HH:mm:ss.SSS",
"yyyy.MM.dd'T'HH:mm:ss.SSS",
"yyyy-MM-dd'T'HH:mm:ss",
"yyyy/MM/dd'T'HH:mm:ss",
"yyyy.MM.dd'T'HH:mm:ss",
"yyyy-MM-dd HH:mm:ssZZ",
"yyyy/MM/dd HH:mm:ssZZ",
"yyyy.MM.dd HH:mm:ssZZ",
"yyyy-MM-dd'T'HH:mm:ssZZ",
"yyyy/MM/dd'T'HH:mm:ssZZ",
"yyyy.MM.dd'T'HH:mm:ssZZ",
"yyyy.MM.dd'T'HH:mm:ss"
};
protected static String host;
protected static String port;
Expand All @@ -93,7 +109,7 @@ public abstract class AbstractCsvTool {
protected static String timeFormat;
protected static Session session;

AbstractCsvTool() {}
public AbstractCsvTool() {}

protected static String checkRequiredArg(String arg, String name, CommandLine commandLine)
throws ArgsErrorException {
Expand All @@ -114,16 +130,13 @@ protected static void setTimeZone() throws IoTDBConnectionException, StatementEx
zoneId = ZoneId.of(session.getTimeZone());
}

protected static void parseBasicParams(CommandLine commandLine, ConsoleReader reader)
protected static void parseBasicParams(CommandLine commandLine)
throws ArgsErrorException, IOException {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine);
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine);

password = commandLine.getOptionValue(PASSWORD_ARGS);
if (password == null) {
password = reader.readLine("please input your password:", '\0');
}
}

protected static boolean checkTimeFormat() {
Expand Down Expand Up @@ -183,9 +196,40 @@ protected static Options createNewOptions() {
.optionalArg(true)
.argName(PASSWORD_NAME)
.hasArg()
.desc("Password (optional)")
.desc("Password (required)")
.build();
options.addOption(opPassword);
return options;
}

/**
* write data to CSV file.
*
* @param headerNames the header names of CSV file
* @param records the records of CSV file
* @param filePath the directory to save the file
*/
public static Boolean writeCsvFile(
@Nullable List<String> headerNames, List<List<Object>> records, String filePath) {
try {
CSVPrinter printer =
CSVFormat.DEFAULT
.withFirstRecordAsHeader()
.withEscape('\\')
.withQuoteMode(QuoteMode.NONE)
.print(new PrintWriter(filePath));
if (headerNames != null) {
printer.printRecord(headerNames);
}
for (List record : records) {
printer.printRecord(record);
}
printer.flush();
printer.close();
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
}
Loading