Skip to content

Commit

Permalink
Add micro and nano timestamp precision (#285)
Browse files Browse the repository at this point in the history
* add micro and nano precision
  • Loading branch information
1160300922 authored and Jialin Qiao committed Jul 23, 2019
1 parent 5f086ed commit ab7d2db
Show file tree
Hide file tree
Showing 14 changed files with 494 additions and 30 deletions.
Expand Up @@ -82,7 +82,7 @@ public abstract class AbstractClient {
protected static final String SHOW_METADATA_COMMAND = "show timeseries";
protected static final int MAX_HELP_CONSOLE_WIDTH = 88;
protected static final String TIMESTAMP_STR = "Time";
protected static final int ISO_DATETIME_LEN = 26;
protected static final int ISO_DATETIME_LEN = 35;
protected static final String IMPORT_CMD = "import";
private static final String NEED_NOT_TO_PRINT_TIMESTAMP = "AGGREGATION";
private static final String DEFAULT_TIME_FORMAT = "default";
Expand All @@ -92,6 +92,8 @@ public abstract class AbstractClient {
protected static int maxTimeLength = ISO_DATETIME_LEN;
protected static int maxValueLength = 15;
protected static boolean isQuit = false;
protected static String TIMESTAMP_PRECISION = "ms";

/**
* control the width of columns for 'show timeseries path' and 'show storage group'.
* <p>
Expand Down Expand Up @@ -187,7 +189,8 @@ public static void output(ResultSet res, boolean printToConsole, ZoneId zoneId)

boolean isShow = res instanceof IoTDBMetadataResultSet;
if (!isShow && resultSetMetaData.getColumnTypeName(0) != null) {
printTimestamp = !res.getMetaData().getColumnTypeName(0).equalsIgnoreCase(NEED_NOT_TO_PRINT_TIMESTAMP);
printTimestamp = !res.getMetaData().getColumnTypeName(0)
.equalsIgnoreCase(NEED_NOT_TO_PRINT_TIMESTAMP);
}
if (res instanceof IoTDBQueryResultSet) {
printTimestamp = printTimestamp && !((IoTDBQueryResultSet) res).isIgnoreTimeStamp();
Expand Down Expand Up @@ -219,6 +222,10 @@ public static void output(ResultSet res, boolean printToConsole, ZoneId zoneId)
printCount(isShow, res, cnt);
}

protected static String getTimestampPrecision() {
return TIMESTAMP_PRECISION;
}

protected static void printCount(boolean isShow, ResultSet res, int cnt) throws SQLException {
if (isShow) {
int type = res.getType();
Expand Down Expand Up @@ -341,6 +348,50 @@ protected static Options createOptions() {
return options;
}

public static String parseLongToDateWithPrecision(DateTimeFormatter formatter,
long timestamp, ZoneId zoneid, String timestampPrecision) {
if (timestampPrecision.equals("ms")) {
long integerofDate = timestamp / 1000;
String digits = Long.toString(timestamp % 1000);
ZonedDateTime dateTime = ZonedDateTime
.ofInstant(Instant.ofEpochSecond(integerofDate), zoneid);
String datetime = dateTime.format(formatter);
int length = digits.length();
if (length != 3) {
for (int i = 0; i < 3 - length; i++) {
digits = "0" + digits;
}
}
return datetime.substring(0, 19) + "." + digits + datetime.substring(19);
} else if (timestampPrecision.equals("us")) {
long integerofDate = timestamp / 1000_000;
String digits = Long.toString(timestamp % 1000_000);
ZonedDateTime dateTime = ZonedDateTime
.ofInstant(Instant.ofEpochSecond(integerofDate), zoneid);
String datetime = dateTime.format(formatter);
int length = digits.length();
if (length != 6) {
for (int i = 0; i < 6 - length; i++) {
digits = "0" + digits;
}
}
return datetime.substring(0, 19) + "." + digits + datetime.substring(19);
} else {
long integerofDate = timestamp / 1000_000_000L;
String digits = Long.toString(timestamp % 1000_000_000L);
ZonedDateTime dateTime = ZonedDateTime
.ofInstant(Instant.ofEpochSecond(integerofDate), zoneid);
String datetime = dateTime.format(formatter);
int length = digits.length();
if (length != 9) {
for (int i = 0; i < 9 - length; i++) {
digits = "0" + digits;
}
}
return datetime.substring(0, 19) + "." + digits + datetime.substring(19);
}
}

private static String formatDatetime(long timestamp, ZoneId zoneId) {
ZonedDateTime dateTime;
switch (timeFormat) {
Expand All @@ -349,8 +400,8 @@ private static String formatDatetime(long timestamp, ZoneId zoneId) {
return Long.toString(timestamp);
case DEFAULT_TIME_FORMAT:
case "iso8601":
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), zoneId);
return dateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
return parseLongToDateWithPrecision(
DateTimeFormatter.ISO_OFFSET_DATE_TIME, timestamp, zoneId, getTimestampPrecision());
default:
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), zoneId);
return dateTime.format(DateTimeFormatter.ofPattern(timeFormat));
Expand Down Expand Up @@ -712,11 +763,11 @@ protected static void executeQuery(IoTDBConnection connection, String cmd) {
enum OperationResult {
STOP_OPER, CONTINUE_OPER, NO_OPER
}

protected static void printf(String format, Object ... args) {
SCREEN_PRINTER.printf(format, args);
}

protected static void print(String msg) {
SCREEN_PRINTER.println(msg);
}
Expand Down
2 changes: 2 additions & 0 deletions client/src/main/java/org/apache/iotdb/cli/client/Client.java
Expand Up @@ -130,6 +130,8 @@ private static void receiveCommands(ConsoleReader reader) throws TException, IOE
String s;
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
TIMESTAMP_PRECISION = properties.getTimestampPrecision();

displayLogo(properties.getVersion());
println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
Expand Down
Expand Up @@ -135,6 +135,8 @@ private static void receiveCommands(Scanner scanner) throws TException {
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password)) {
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
TIMESTAMP_PRECISION = properties.getTimestampPrecision();

displayLogo(properties.getVersion());
println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
Expand Down
Expand Up @@ -82,13 +82,15 @@ public void testGetServerProperties() throws IoTDBSQLException, TException {
add("min_time");
}
};
final String timestampPrecision = "ms";
when(client.getProperties())
.thenReturn(new ServerProperties(version, supportedAggregationTime));
.thenReturn(new ServerProperties(version, supportedAggregationTime, timestampPrecision));
connection.client = client;
assertEquals(connection.getServerProperties().getVersion(), version);
for (int i = 0; i < supportedAggregationTime.size(); i++) {
assertEquals(connection.getServerProperties().getSupportedTimeAggregationOperations().get(i),
supportedAggregationTime.get(i));
}
assertEquals(connection.getServerProperties().getTimestampPrecision(), timestampPrecision);
}
}
9 changes: 9 additions & 0 deletions server/iotdb/conf/iotdb-engine.properties
Expand Up @@ -53,6 +53,15 @@ flush_wal_threshold=10000
# Set this parameter to 0 may slow down the ingestion on slow disk.
force_wal_period_in_ms=10


####################
### Timestamp Precision Configuration
####################
# Use this value to set timestamp precision as "ms", "us" or "ns".
# Once the precision is been set, it can not be changed.
timestamp_precision=ms


####################
### Directory Configuration
####################
Expand Down
34 changes: 31 additions & 3 deletions server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
Expand Up @@ -71,6 +71,11 @@ public class IoTDBConfig {
*/
private int flushWalThreshold = 10000;

/**
* this variable set timestamp precision as millisecond, microsecond or nanosecond
*/
private String timestampPrecision = "ms";

/**
* The cycle when write ahead log is periodically forced to be written to disk(in milliseconds) If
* set this parameter to 0 it means call outputStream.force(true) after every each insert
Expand All @@ -93,6 +98,11 @@ public class IoTDBConfig {
*/
private String systemDir = "data/system";

/**
* Schema directory, including storage set of values.
*/
private String schemaDir = "data/system/schema";

/**
* Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"};
*/
Expand Down Expand Up @@ -240,6 +250,7 @@ private void formulateFolders() {
List<String> dirs = new ArrayList<>();
dirs.add(baseDir);
dirs.add(systemDir);
dirs.add(schemaDir);
dirs.add(walFolder);
dirs.add(indexFileDir);
dirs.addAll(Arrays.asList(dataDirs));
Expand All @@ -258,10 +269,11 @@ private void formulateFolders() {
}
baseDir = dirs.get(0);
systemDir = dirs.get(1);
walFolder = dirs.get(2);
indexFileDir = dirs.get(3);
schemaDir = dirs.get(2);
walFolder = dirs.get(3);
indexFileDir = dirs.get(4);
for (int i = 0; i < dataDirs.length; i++) {
dataDirs[i] = dirs.get(i + 4);
dataDirs[i] = dirs.get(i + 5);
}
}

Expand Down Expand Up @@ -303,6 +315,14 @@ void setRpcPort(int rpcPort) {
this.rpcPort = rpcPort;
}

public void setTimestampPrecision(String timestampPrecision) {
this.timestampPrecision = timestampPrecision;
}

public String getTimestampPrecision() {
return timestampPrecision;
}

public boolean isEnableWal() {
return enableWal;
}
Expand Down Expand Up @@ -335,6 +355,14 @@ void setSystemDir(String systemDir) {
this.systemDir = systemDir;
}

public String getSchemaDir() {
return schemaDir;
}

void setSchemaDir(String schemaDir) {
this.schemaDir = schemaDir;
}

public String getWalFolder() {
return walFolder;
}
Expand Down
@@ -0,0 +1,94 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iotdb.db.conf;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Properties;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class IoTDBConfigCheck {

// this file is located in data/system/schema/system_properties.
// If user delete folder "data", system_properties can reset.
public static final String PROPERTIES_FILE_NAME = "system.properties";
public static final String SCHEMA_DIR =
IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
private static final IoTDBConfigCheck INSTANCE = new IoTDBConfigCheck();
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
private Properties properties = new Properties();
// this is a initial parameter.
private static String TIMESTAMP_PRECISION = "ms";

public static final IoTDBConfigCheck getInstance() {
return IoTDBConfigCheck.INSTANCE;
}

public void checkConfig() {
TIMESTAMP_PRECISION = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
createDir(SCHEMA_DIR);
checkFile(SCHEMA_DIR);
logger.info("System configuration is ok.");
}

public void createDir(String filepath) {
File dir = new File(filepath);
if (!dir.exists()) {
dir.mkdirs();
logger.info(" {} dir has been created.", SCHEMA_DIR);
}
}

public void checkFile(String filepath) {
// create file : read timestamp precision from engine.properties, create system_properties.txt
// use output stream to write timestamp precision to file.
File file = new File(filepath + File.separator + PROPERTIES_FILE_NAME);
try {
if (!file.exists()) {
file.createNewFile();
logger.info(" {} has been created.", file.getAbsolutePath());
try (FileOutputStream outputStream = new FileOutputStream(file.toString())) {
properties.setProperty("timestamp_precision", TIMESTAMP_PRECISION);
properties.store(outputStream, "System properties:");
}
}
} catch (IOException e) {
logger.error("Can not create {}.", file.getAbsolutePath(), e);
}
// get existed properties from system_properties.txt
File inputFile = new File(filepath + File.separator + PROPERTIES_FILE_NAME);
try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_ENCODING));
if (!properties.getProperty("timestamp_precision").equals(TIMESTAMP_PRECISION)) {
logger.error("Wrong timestamp precision, please set as: " + properties
.getProperty("timestamp_precision") + " !");
System.exit(-1);
}
} catch (IOException e) {
logger.error("Load system.properties from {} failed.", file.getAbsolutePath(), e);
}
}
}


Expand Up @@ -123,6 +123,9 @@ private void loadProps() {
conf.setRpcPort(Integer.parseInt(properties.getProperty("rpc_port",
Integer.toString(conf.getRpcPort()))));

conf.setTimestampPrecision(properties.getProperty("timestamp_precision",
conf.getTimestampPrecision()));

conf.setEnableParameterAdapter(
Boolean.parseBoolean(properties.getProperty("enable_parameter_adapter",
Boolean.toString(conf.isEnableParameterAdapter()))));
Expand All @@ -136,6 +139,8 @@ private void loadProps() {

conf.setSystemDir(FilePathUtils.regularizePath(conf.getBaseDir()) + "system");

conf.setSchemaDir(FilePathUtils.regularizePath(conf.getSystemDir()) + "schema");

conf.setDataDirs(properties.getProperty("data_dirs", conf.getDataDirs()[0])
.split(","));

Expand Down

0 comments on commit ab7d2db

Please sign in to comment.