diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index 341eb178d0783..aa3472326d2f9 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -36,6 +36,8 @@ public interface Record { String RECORD_ERROR_SUPPORT = "talend.component.record.error.support"; + String RECORD_NULLABLE_CHECK = "talend.component.record.nullable.check"; + /** * @return the schema of this record. */ diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/MappingUtils.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/MappingUtils.java index e817ff37bf94d..66da64af7ff1b 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/MappingUtils.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/MappingUtils.java @@ -179,8 +179,9 @@ public static Object mapString(final Class expected, final String value) if ("null".equalsIgnoreCase(value.trim())) { return null; } - // - final boolean isNumeric = value.chars().allMatch(Character::isDigit); + + // A timestamp can be negative if before 1/1/1970, so need to consider '-' as first character. + boolean isNumeric = !value.isEmpty() && (value.charAt(0) == '-' ? value.substring(1) : value).chars().allMatch(Character::isDigit); if (ZonedDateTime.class == expected) { if (isNumeric) { return ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.valueOf(value)), UTC); @@ -191,7 +192,6 @@ public static Object mapString(final Class expected, final String value) if (Date.class == expected) { if (isNumeric) { return Date.from(Instant.ofEpochMilli(Long.valueOf(value))); - } else { return Date.from(ZonedDateTime.parse(value).toInstant()); } diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 5ca694666e359..290a65069c92c 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -123,6 +123,9 @@ public Builder withNewSchema(final Schema newSchema) { // Entry creation can be optimized a bit but recent GC should not see it as a big deal public static class BuilderImpl implements Builder { + private final boolean skipNullCheck = + Boolean.parseBoolean(System.getProperty(Record.RECORD_NULLABLE_CHECK, "false")); + private final Map values = new HashMap<>(8); private final OrderedMap entries; @@ -327,7 +330,7 @@ private Schema.Entry validateTypeAgainstProvidedSchema(final String name, final "Entry '" + entry.getOriginalFieldName() + "' expected to be a " + entry.getType() + ", got a " + type); } - if (value == null && !entry.isNullable()) { + if (!skipNullCheck && value == null && !entry.isNullable()) { throw new IllegalArgumentException("Entry '" + entry.getOriginalFieldName() + "' is not nullable"); } return entry; @@ -357,7 +360,7 @@ public Record build() { .filter(it -> !it.isNullable() && !values.containsKey(it.getName())) .map(Schema.Entry::getName) .collect(joining(", ")); - if (!missing.isEmpty()) { + if (!skipNullCheck && !missing.isEmpty()) { throw new IllegalArgumentException("Missing entries: " + missing); } @@ -587,10 +590,15 @@ private Builder append(final Schema.Entry entry, final T value) { } else { realEntry = entry; } - if (value != null) { + + if (skipNullCheck) { values.put(realEntry.getName(), value); - } else if (!realEntry.isNullable()) { - throw new IllegalArgumentException(realEntry.getName() + " is not nullable but got a null value"); + } else { + if (value != null) { + values.put(realEntry.getName(), value); + } else if (!realEntry.isNullable()) { + throw new IllegalArgumentException(realEntry.getName() + " is not nullable but got a null value"); + } } if (this.entries != null) { diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/MappingUtilsTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/MappingUtilsTest.java index 01a27610c9ea3..2f1f9afbcd53a 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/MappingUtilsTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/MappingUtilsTest.java @@ -28,8 +28,13 @@ import java.time.ZonedDateTime; import java.util.Date; import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class MappingUtilsTest { @@ -105,4 +110,74 @@ void coerce() { // incompatible mapping: fail assertThrows(IllegalArgumentException.class, () -> MappingUtils.coerce(List.class, 123, name)); } + + + @ParameterizedTest + @MethodSource("mapStringProvider") + void mapString(final Class expectedType, final String inputValue, final Object expectedResult) { + Object mapped = MappingUtils.coerce(expectedType, inputValue, "::testing::mapString"); + if(expectedResult instanceof byte[]){ + Assertions.assertArrayEquals((byte[])expectedResult, (byte[])mapped); + } + else { + Assertions.assertEquals(expectedResult, mapped); + } + } + + static Stream mapStringProvider() { + final ZonedDateTime zdtAfterEpochWithNano = ZonedDateTime.of( + 2024, 7, 15, // Année, mois, jour + 14, 30, 45, // Heure, minute, seconde + 123_456_789, // Nanosecondes + ZoneId.of("Europe/Paris")); + + final Date dateAfterEpoch = Date.from(zdtAfterEpochWithNano.toInstant()); + + java.time.Instant instant = dateAfterEpoch.toInstant(); + final ZonedDateTime zdtAfterEpochUTCNoNano = instant.atZone(UTC); + + final ZonedDateTime zdtBeforeEpochWithNano = ZonedDateTime.of( + 1930, 7, 15, // Année, mois, jour + 14, 30, 45, // Heure, minute, seconde + 123_456_789, // Nanosecondes + ZoneId.of("Europe/Paris")); + + final Date dateBeforeEpoch = Date.from(zdtBeforeEpochWithNano.toInstant()); + + instant = dateBeforeEpoch.toInstant(); + final ZonedDateTime zdtBeforeEpochUTCNoNano = instant.atZone(UTC); + + + return Stream.of( + // (expectedType, inputValue, expectedResult) + Arguments.of(String.class, "A String", "A String"), + Arguments.of(String.class, "-100", "-100"), + Arguments.of(String.class, "100", "100"), + Arguments.of(Boolean.class, "true", Boolean.TRUE), + Arguments.of(Boolean.class, "tRuE", Boolean.TRUE), + Arguments.of(Boolean.class, "false", Boolean.FALSE), + Arguments.of(Boolean.class, "xxx", Boolean.FALSE), + Arguments.of(Date.class, "null", null), + Arguments.of(ZonedDateTime.class, "2024-07-15T14:30:45.123456789+02:00[Europe/Paris]", + zdtAfterEpochWithNano), + Arguments.of(ZonedDateTime.class, "1721046645123", + zdtAfterEpochUTCNoNano), + Arguments.of(ZonedDateTime.class, "1930-07-15T14:30:45.123456789+01:00[Europe/Paris]", + zdtBeforeEpochWithNano), + Arguments.of(ZonedDateTime.class, "-1245407354877", + zdtBeforeEpochUTCNoNano), + Arguments.of(Character.class, "abcde", 'a'), + Arguments.arguments(byte[].class, "Ojp0ZXN0aW5nOjptYXBTdHJpbmc=", + new byte[]{58, 58, 116, 101, 115, 116, 105, 110, 103, 58, + 58, 109, 97, 112, 83, 116, 114, 105, 110, 103}), + Arguments.of(BigDecimal.class, "123456789123456789", + new BigDecimal("123456789123456789")), + Arguments.of(Integer.class, String.valueOf(Integer.MIN_VALUE), Integer.MIN_VALUE), + Arguments.of(Long.class, String.valueOf(Long.MIN_VALUE), Long.MIN_VALUE), + Arguments.of(Short.class, String.valueOf(Short.MIN_VALUE), Short.MIN_VALUE), + Arguments.of(Byte.class, String.valueOf(Byte.MIN_VALUE), Byte.MIN_VALUE), + Arguments.of(Float.class, String.valueOf(Float.MIN_VALUE), Float.MIN_VALUE), + Arguments.of(Double.class, String.valueOf(Double.MIN_VALUE), Double.MIN_VALUE) + ); + } } \ No newline at end of file diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 0300000cf8e59..da3a12c8f8224 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -232,6 +232,57 @@ void notNullableNullBehavior() { .withString(new SchemaImpl.EntryImpl.BuilderImpl().withNullable(false).withName("test").build(), null)); } + @Test + void disableNullableCheck_no() { + RecordBuilderFactoryImpl recordBuilderFactory = new RecordBuilderFactoryImpl("test"); + Schema.Builder schemaBuilder = recordBuilderFactory.newSchemaBuilder(Type.RECORD); + Schema schema = schemaBuilder.withEntry( + recordBuilderFactory.newEntryBuilder().withType(Type.STRING).withName("c1").withNullable(false).build()) + .withEntry(recordBuilderFactory.newEntryBuilder() + .withName("c2") + .withType(Type.STRING) + .withNullable(true) + .build()) + .build(); + Record.Builder recordBuilder = recordBuilderFactory.newRecordBuilder(schema); + recordBuilder.withString("c1", "v1"); + recordBuilder.build(); + + Schema.Builder schemaBuilder2 = recordBuilderFactory.newSchemaBuilder(Type.RECORD); + Schema schema2 = schemaBuilder2.withEntry( + recordBuilderFactory.newEntryBuilder().withType(Type.STRING).withName("c1").withNullable(false).build()) + .withEntry(recordBuilderFactory.newEntryBuilder() + .withName("c2") + .withType(Type.STRING) + .withNullable(false) + .build()) + .build(); + Record.Builder recordBuilder2 = recordBuilderFactory.newRecordBuilder(schema2); + recordBuilder2.withString("c1", "v1"); + assertThrows(IllegalArgumentException.class, recordBuilder2::build); + } + + @Test + void disableNullableCheck_yes() { + System.setProperty(Record.RECORD_NULLABLE_CHECK, "true"); + RecordBuilderFactoryImpl recordBuilderFactory = new RecordBuilderFactoryImpl("test"); + Schema.Builder schemaBuilder = recordBuilderFactory.newSchemaBuilder(Type.RECORD); + Schema schema = schemaBuilder.withEntry( + recordBuilderFactory.newEntryBuilder().withType(Type.STRING).withName("c1").withNullable(false).build()) + .withEntry(recordBuilderFactory.newEntryBuilder() + .withName("c2") + .withType(Type.STRING) + .withNullable(false) + .build()) + .build(); + Record.Builder recordBuilder = recordBuilderFactory.newRecordBuilder(schema); + recordBuilder.withString("c1", "v1"); + recordBuilder.withString("c2", null); + Record recordOne = recordBuilder.build(); + assertNull(recordOne.getString("c2")); + System.setProperty(Record.RECORD_NULLABLE_CHECK, "false"); + } + @Test void dateTime() { final Schema schema = new SchemaImpl.BuilderImpl() diff --git a/pom.xml b/pom.xml index 1fad9eba66083..78ed0db5433f1 100644 --- a/pom.xml +++ b/pom.xml @@ -174,8 +174,8 @@ 8.0.14 1.2.15 2.0.27 - 9.0.104 - 3.5.10 + 9.0.108 + 3.5.11 6.5.0