From acd03ae4e710f8ee24ba16842b6b09306a030e04 Mon Sep 17 00:00:00 2001 From: Thomas Calmant Date: Wed, 7 Feb 2024 19:26:20 +0100 Subject: [PATCH] Enhanced parsing of date & time Allow locale-based date time strings Signed-off-by: Thomas Calmant --- .../device/factory/LocaleUtils.java | 59 +++++++++ .../southbound/device/factory/ValueType.java | 59 ++------- .../factory/dto/DeviceMappingOptionsDTO.java | 9 ++ .../factory/impl/FactoryParserHandler.java | 117 +++++++++++++----- .../factory/impl/RecordHandlingTest.java | 98 +++++++++++++++ 5 files changed, 267 insertions(+), 75 deletions(-) create mode 100644 southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/LocaleUtils.java diff --git a/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/LocaleUtils.java b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/LocaleUtils.java new file mode 100644 index 000000000..e9cae37b8 --- /dev/null +++ b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/LocaleUtils.java @@ -0,0 +1,59 @@ +/********************************************************************* +* Copyright (c) 2024 Kentyou. +* +* This program and the accompanying materials are made +* available under the terms of the Eclipse Public License 2.0 +* which is available at https://www.eclipse.org/legal/epl-2.0/ +* +* SPDX-License-Identifier: EPL-2.0 +* +* Contributors: +* Thomas Calmant (Kentyou) - initial implementation +**********************************************************************/ +package org.eclipse.sensinact.gateway.southbound.device.factory; + +import java.util.Locale; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility methods to find a locale + */ +public class LocaleUtils { + + private static final Logger logger = LoggerFactory.getLogger(LocaleUtils.class); + + /** + * Try to find the locale from the given string. + * + * The string can be a language code, a language code and a country code or a + * language code, a country code and a variant. + * Parts are separated by an underscore character, for example: fr_FR + * + * @param strLocale Locale string + * @return + */ + public static Locale fromString(final String strLocale) { + if (strLocale == null || strLocale.isBlank()) { + return null; + } + + final String[] parts = strLocale.split("_"); + switch (parts.length) { + case 1: + return new Locale(parts[0]); + + case 2: + return new Locale(parts[0], parts[1]); + + case 3: + return new Locale(parts[0], parts[1], parts[2]); + + default: + // Invalid locale string + logger.warn("Unhandled number locale {}", strLocale); + return null; + } + } +} diff --git a/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/ValueType.java b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/ValueType.java index 959f85be3..c68dd6aea 100644 --- a/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/ValueType.java +++ b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/ValueType.java @@ -26,16 +26,12 @@ import java.util.stream.Stream; import org.eclipse.sensinact.gateway.southbound.device.factory.dto.DeviceMappingOptionsDTO; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public enum ValueType { /** * Represent available types */ - AS_IS("any", null, null), - STRING("string", (v, o) -> String.valueOf(v), String.class), - CHAR("char", (v, o) -> { + AS_IS("any", null, null), STRING("string", (v, o) -> String.valueOf(v), String.class), CHAR("char", (v, o) -> { if (v instanceof Character) { return (Character) v; } else if (v instanceof CharSequence) { @@ -55,8 +51,7 @@ public enum ValueType { } else { return null; } - }, Character.class), - BOOLEAN("boolean", (v, o) -> { + }, Character.class), BOOLEAN("boolean", (v, o) -> { if (v instanceof Boolean) { return (Boolean) v; } else if (v instanceof Number) { @@ -66,32 +61,25 @@ public enum ValueType { } else { return null; } - }, Boolean.class), - BYTE("byte", (v, o) -> { + }, Boolean.class), BYTE("byte", (v, o) -> { final Number parsed = parseNumber(v, true, o); return parsed != null ? parsed.byteValue() : null; - }, Byte.class), - SHORT("short", (v, o) -> { + }, Byte.class), SHORT("short", (v, o) -> { final Number parsed = parseNumber(v, true, o); return parsed != null ? parsed.shortValue() : null; - }, Short.class), - INT("int", (v, o) -> { + }, Short.class), INT("int", (v, o) -> { final Number parsed = parseNumber(v, true, o); return parsed != null ? parsed.intValue() : null; - }, Integer.class), - LONG("long", (v, o) -> { + }, Integer.class), LONG("long", (v, o) -> { final Number parsed = parseNumber(v, true, o); return parsed != null ? parsed.longValue() : null; - }, Long.class), - FLOAT("float", (v, o) -> { + }, Long.class), FLOAT("float", (v, o) -> { final Number parsed = parseNumber(v, false, o); return parsed != null ? parsed.floatValue() : null; - }, Float.class), - DOUBLE("double", (v, o) -> { + }, Float.class), DOUBLE("double", (v, o) -> { final Number parsed = parseNumber(v, false, o); return parsed != null ? parsed.doubleValue() : null; - }, Double.class), - ANY_ARRAY("any[]", (v, o) -> asList(v, o, AS_IS), List.class), + }, Double.class), ANY_ARRAY("any[]", (v, o) -> asList(v, o, AS_IS), List.class), STRING_ARRAY("string[]", (v, o) -> asList(v, o, STRING), List.class), CHAR_ARRAY("char[]", (v, o) -> asList(v, o, CHAR), List.class), BOOLEAN_ARRAY("boolean[]", (v, o) -> asList(v, o, BOOLEAN), List.class), @@ -102,8 +90,6 @@ public enum ValueType { FLOAT_ARRAY("float[]", (v, o) -> asList(v, o, FLOAT), List.class), DOUBLE_ARRAY("double[]", (v, o) -> asList(v, o, DOUBLE), List.class); - private static final Logger logger = LoggerFactory.getLogger(ValueType.class); - /** * String representation */ @@ -126,28 +112,8 @@ public enum ValueType { * @return The number format for the given locale or null */ private static NumberFormat getNumberFormat(final DeviceMappingOptionsDTO options, boolean integers) { - final String strLocale = options.numbersLocale; - if (strLocale == null || strLocale.isBlank()) { - return null; - } - - final String[] parts = strLocale.split("_"); - final Locale locale; - switch (parts.length) { - case 1: - locale = new Locale(parts[0]); - break; - - case 2: - locale = new Locale(parts[0], parts[1]); - break; - - case 3: - locale = new Locale(parts[0], parts[1], parts[2]); - break; - - default: - logger.warn("Unhandled number locale {}", strLocale); + final Locale locale = LocaleUtils.fromString(options.numbersLocale); + if (locale == null) { return null; } @@ -220,7 +186,8 @@ private static Number parseNumber(final Object value, final boolean expectIntege * @param converter Function to convert any input to the value type * @param javaClass Java class represented by the value type */ - ValueType(final String strRepr, final BiFunction converter, final Class javaClass) { + ValueType(final String strRepr, final BiFunction converter, + final Class javaClass) { this.repr = strRepr; this.converter = converter; this.javaClass = javaClass != null ? javaClass : Object.class; diff --git a/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/dto/DeviceMappingOptionsDTO.java b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/dto/DeviceMappingOptionsDTO.java index f86266f30..de30a1382 100644 --- a/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/dto/DeviceMappingOptionsDTO.java +++ b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/dto/DeviceMappingOptionsDTO.java @@ -30,6 +30,15 @@ public class DeviceMappingOptionsDTO { @JsonProperty("format.datetime") public String formatDateTime; + @JsonProperty("format.datetime.locale") + public String formatDateTimeLocale; + + @JsonProperty("format.date.style") + public String formatDateStyle; + + @JsonProperty("format.time.style") + public String formatTimeStyle; + @JsonProperty("datetime.timezone") public String dateTimezone; diff --git a/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/FactoryParserHandler.java b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/FactoryParserHandler.java index d5861e9b1..38fba061c 100644 --- a/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/FactoryParserHandler.java +++ b/southbound/device-factory/device-factory-core/src/main/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/FactoryParserHandler.java @@ -23,6 +23,7 @@ import java.time.ZoneOffset; import java.time.chrono.ChronoZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.time.temporal.UnsupportedTemporalTypeException; @@ -30,6 +31,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -53,6 +55,7 @@ import org.eclipse.sensinact.gateway.southbound.device.factory.IPlaceHolderKeys; import org.eclipse.sensinact.gateway.southbound.device.factory.IResourceMapping; import org.eclipse.sensinact.gateway.southbound.device.factory.InvalidResourcePathException; +import org.eclipse.sensinact.gateway.southbound.device.factory.LocaleUtils; import org.eclipse.sensinact.gateway.southbound.device.factory.MissingParserException; import org.eclipse.sensinact.gateway.southbound.device.factory.ParserException; import org.eclipse.sensinact.gateway.southbound.device.factory.RecordPath; @@ -799,12 +802,15 @@ private Instant computeTimestamp(final String provider, final IDeviceMappingReco } final ZoneId timezone = getTimezone(configuration.mappingOptions.dateTimezone); + final Locale locale = LocaleUtils.fromString(configuration.mappingOptions.formatDateTimeLocale); final IResourceMapping dateTimePath = placeholders.get(KEY_DATETIME); if (dateTimePath != null) { final String strDateTime = getFieldString(record, dateTimePath, options); if (strDateTime != null && !strDateTime.isBlank()) { - return parseDateTime(strDateTime, timezone, configuration.mappingOptions.formatDateTime); + final TemporalAccessor parsedDateTime = parseDateTime(strDateTime, configuration.mappingOptions, + locale); + return extractDateTime(parsedDateTime, timezone); } } @@ -813,7 +819,7 @@ private Instant computeTimestamp(final String provider, final IDeviceMappingReco if (datePath != null) { final String strDate = getFieldString(record, datePath, options); if (strDate != null && !strDate.isBlank()) { - date = parseDate(strDate, configuration.mappingOptions.formatDate); + date = parseDate(strDate, configuration.mappingOptions, locale); } } @@ -822,7 +828,7 @@ private Instant computeTimestamp(final String provider, final IDeviceMappingReco if (timePath != null) { final String strTime = getFieldString(record, timePath, options); if (strTime != null && !strTime.isBlank()) { - time = parseTime(strTime, date, timezone, configuration.mappingOptions.formatTime); + time = parseTime(strTime, date, timezone, configuration.mappingOptions, locale); } } @@ -878,31 +884,45 @@ private OffsetTime extractTime(final TemporalAccessor parsedTime, final LocalDat second = 0; } + int nanoOfSecond; + try { + nanoOfSecond = parsedTime.get(ChronoField.NANO_OF_SECOND); + } catch (Exception e) { + nanoOfSecond = 0; + } + ZoneOffset offset; try { offset = ZoneOffset.ofTotalSeconds(parsedTime.get(ChronoField.OFFSET_SECONDS)); } catch (UnsupportedTemporalTypeException e) { if (expectedDate != null) { - offset = timezone.getRules().getOffset(expectedDate.atTime(hour, minute, second)); + offset = timezone.getRules().getOffset(expectedDate.atTime(hour, minute, second, nanoOfSecond)); } else { offset = timezone.getRules().getOffset(Instant.now()); } } - return OffsetTime.of(hour, minute, second, 0, offset); + return OffsetTime.of(hour, minute, second, nanoOfSecond, offset); } /** * Parses a date string * - * @param strDate Date string - * @param formatDate Custom parsing format (can be null) + * @param strDate Date string + * @param options Device mapping options + * @param locale Configured locale * @return The parsed date */ - private LocalDate parseDate(String strDate, String formatDate) { + private LocalDate parseDate(final String strDate, final DeviceMappingOptionsDTO options, final Locale locale) { DateTimeFormatter format = DateTimeFormatter.ISO_LOCAL_DATE; - if (formatDate != null && !formatDate.isBlank()) { - format = DateTimeFormatter.ofPattern(formatDate); + if (options.formatDate != null && !options.formatDate.isBlank()) { + format = DateTimeFormatter.ofPattern(options.formatDate); + } else if (options.formatDateStyle != null && !options.formatDateStyle.isBlank()) { + format = DateTimeFormatter.ofLocalizedDate(FormatStyle.valueOf(options.formatDateStyle.toUpperCase())); + } + + if (locale != null) { + format = format.withLocale(locale); } return extractDate(format.parse(strDate)); @@ -911,35 +931,75 @@ private LocalDate parseDate(String strDate, String formatDate) { /** * Parses a time string * - * @param strTime Time string - * @param timezone Fallback timezone - * @param formatTime Custom parsing format (can be null) - * @return The parsed date + * @param strTime Time string + * @param expectedDate Expected date of the time (today if null) + * @param timezone Fallback timezone + * @param options Device mapping options + * @param locale Configured locale + * @return The parsed time at the expected date or today */ - private OffsetTime parseTime(String strTime, LocalDate expectedDate, ZoneId timezone, String formatTime) { + private OffsetTime parseTime(final String strTime, final LocalDate expectedDate, final ZoneId timezone, + final DeviceMappingOptionsDTO options, final Locale locale) { DateTimeFormatter format = DateTimeFormatter.ISO_OFFSET_TIME; - if (formatTime != null && !formatTime.isBlank()) { - format = DateTimeFormatter.ofPattern(formatTime); + if (options.formatTime != null && !options.formatTime.isBlank()) { + format = DateTimeFormatter.ofPattern(options.formatTime); + } else if (options.formatTimeStyle != null && !options.formatTimeStyle.isBlank()) { + format = DateTimeFormatter.ofLocalizedDate(FormatStyle.valueOf(options.formatTimeStyle.toUpperCase())); + } + + if (locale != null) { + format = format.withLocale(locale); } return extractTime(format.parse(strTime), expectedDate, timezone); } /** - * Parses a date/time string + * Parses a date time according to mapping options * - * @param strDateTime Date/time string - * @param timezone Fallback timezone - * @param formatDateTime Custom parsing format (can be null) - * @return The parsed date as an instant + * @param strDateTime String value of the date time + * @param options Mapping options + * @param locale Configured locale + * @return The parsed date time */ - private Instant parseDateTime(String strDateTime, ZoneId timezone, String formatDateTime) { + private TemporalAccessor parseDateTime(final String strDateTime, final DeviceMappingOptionsDTO options, + final Locale locale) { DateTimeFormatter format = DateTimeFormatter.ISO_DATE_TIME; + + final String formatDateTime = options.formatDateTime; if (formatDateTime != null && !formatDateTime.isBlank()) { format = DateTimeFormatter.ofPattern(formatDateTime); + } else { + String formatDateStyle = options.formatDateStyle; + String formatTimeStyle = options.formatTimeStyle; + + if (formatTimeStyle == null || formatTimeStyle.isBlank()) { + formatTimeStyle = formatDateStyle; + } else if (formatDateStyle == null || formatDateStyle.isBlank()) { + formatDateStyle = formatTimeStyle; + } + + if (formatTimeStyle != null && !formatTimeStyle.isBlank()) { + format = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.valueOf(formatDateStyle.toUpperCase()), + FormatStyle.valueOf(formatTimeStyle.toUpperCase())); + } + } + + if (locale != null) { + format = format.withLocale(locale); } - final TemporalAccessor parsedTime = format.parse(strDateTime); + return format.parse(strDateTime); + } + + /** + * Extract date and time from a parsed temporal accessor + * + * @param parsedTime Parsed date time + * @param timezone Fallback timezone + * @return The parsed date as an instant + */ + private Instant extractDateTime(final TemporalAccessor parsedTime, final ZoneId timezone) { final LocalDate date = extractDate(parsedTime); final OffsetTime offsetTime = extractTime(parsedTime, date, timezone); final OffsetDateTime dateTime = OffsetDateTime.of(date, offsetTime.toLocalTime(), offsetTime.getOffset()); @@ -955,15 +1015,14 @@ private Instant parseDateTime(String strDateTime, ZoneId timezone, String format */ private Instant convertTimestamp(final String provider, final Long timestamp) { - int currentLogMs = (int) Math.log10(System.currentTimeMillis()); - int currentLogNs = (int) Math.log10(System.nanoTime()); + int log10ms = (int) Math.log10(System.currentTimeMillis()); int timestampLog = (int) Math.log10(timestamp); - if (timestampLog == currentLogMs) { + if (timestampLog == log10ms) { return Instant.ofEpochMilli(timestamp); - } else if (timestampLog == currentLogMs - 3) { + } else if (timestampLog == log10ms - 3) { return Instant.ofEpochSecond(timestamp); - } else if (timestampLog == currentLogNs) { + } else if (timestampLog == log10ms + 6) { return Instant.EPOCH.plusNanos(timestamp); } else { logger.warn("Can't parse timestamp {} for provider {}", timestamp, provider); diff --git a/southbound/device-factory/device-factory-core/src/test/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/RecordHandlingTest.java b/southbound/device-factory/device-factory-core/src/test/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/RecordHandlingTest.java index 03d170270..f26b1a65f 100644 --- a/southbound/device-factory/device-factory-core/src/test/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/RecordHandlingTest.java +++ b/southbound/device-factory/device-factory-core/src/test/java/org/eclipse/sensinact/gateway/southbound/device/factory/impl/RecordHandlingTest.java @@ -16,10 +16,18 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.fail; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.eclipse.sensinact.core.annotation.dto.NullAction; import org.eclipse.sensinact.core.push.DataUpdate; @@ -326,4 +334,94 @@ void testVariableService() throws Exception { // Test values assertEquals(42, getResourceValue("provider", "svc", "rc").value); } + + @Test + void testTimestamp() throws Exception { + final DeviceMappingConfigurationDTO config = prepareConfig(); + + // Set a record + final Instant now = Instant.now(); + final Map record = new HashMap<>(); + record.put("provider", "provider"); + record.put("timestamp_s", now.getEpochSecond()); + record.put("timestamp_ms", now.toEpochMilli()); + final long timestampNs = TimeUnit.SECONDS.toNanos(now.getEpochSecond()) + now.getNano(); + record.put("timestamp_ns", timestampNs); + parser.setRecords(record); + + // Auto-compute timestamp format: seconds + config.mapping.put("@provider", "provider"); + config.mapping.put("data/val", null); + config.mapping.put("@timestamp", "timestamp_s"); + deviceMapper.handle(config, Map.of(), new byte[0]); + GenericDto dto = getResourceValue("provider", "data", "val"); + assertEquals(Instant.ofEpochSecond(now.getEpochSecond()), dto.timestamp); + + // Auto-compute timestamp format: milliseconds + bulks.clear(); + config.mapping.put("@timestamp", "timestamp_ms"); + deviceMapper.handle(config, Map.of(), new byte[0]); + dto = getResourceValue("provider", "data", "val"); + assertEquals(Instant.ofEpochMilli(now.toEpochMilli()), dto.timestamp); + + // Auto-compute timestamp format: nanoseconds + bulks.clear(); + config.mapping.put("@timestamp", "timestamp_ns"); + deviceMapper.handle(config, Map.of(), new byte[0]); + dto = getResourceValue("provider", "data", "val"); + assertEquals(now, dto.timestamp); + } + + @Test + void testDateTime() throws Exception { + final DeviceMappingConfigurationDTO config = prepareConfig(); + + // Set a record + final Instant now = Instant.now(); + final OffsetDateTime date = now.atOffset(ZoneOffset.ofHours(1)); + final DateTimeFormatter formatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.LONG, FormatStyle.MEDIUM); + final String strPattern = "MM dd, yyyy - Q - hh:mm a - s.n - X"; + + final Map record = new HashMap<>(); + record.put("provider", "provider"); + record.put("iso", date.toString()); + record.put("custom", date.format(DateTimeFormatter.ofPattern(strPattern))); + record.put("locale_fr", date.format(formatter.withLocale(Locale.FRANCE))); + record.put("locale_de", date.format(formatter.withLocale(Locale.GERMAN))); + record.put("locale_en", date.format(formatter.withLocale(Locale.UK))); + record.put("locale_zh", date.format(formatter.withLocale(Locale.SIMPLIFIED_CHINESE))); + parser.setRecords(record); + + // ISO format + config.mapping.put("@provider", "provider"); + config.mapping.put("data/val", null); + config.mapping.put("@datetime", "iso"); + config.mappingOptions.formatDateTime = null; + config.mappingOptions.dateTimezone = null; + deviceMapper.handle(config, Map.of(), new byte[0]); + GenericDto dto = getResourceValue("provider", "data", "val"); + assertEquals(now, dto.timestamp); + + // Custom pattern + bulks.clear(); + config.mapping.put("@datetime", "custom"); + config.mappingOptions.formatDateTime = strPattern; + deviceMapper.handle(config, Map.of(), new byte[0]); + dto = getResourceValue("provider", "data", "val"); + assertEquals(now, dto.timestamp); + + // Locale-based date (mix of country and language codes) + for (Locale locale : List.of(Locale.FRANCE, Locale.GERMAN, Locale.UK, Locale.SIMPLIFIED_CHINESE)) { + bulks.clear(); + config.mapping.put("@datetime", "locale_" + locale.getLanguage()); + config.mappingOptions.formatDateTime = null; + config.mappingOptions.formatDateTimeLocale = locale.toString(); + config.mappingOptions.formatDateStyle = "long"; + config.mappingOptions.formatTimeStyle = "medium"; + config.mappingOptions.dateTimezone = "+01"; + deviceMapper.handle(config, Map.of(), new byte[0]); + dto = getResourceValue("provider", "data", "val"); + assertEquals(now.truncatedTo(ChronoUnit.SECONDS), dto.timestamp); + } + } }