Fix ingest timezone parsing (#63876) backport(#79838)

this commit aligns timezone parsing logic with DateFormat.Iso8601. Timezone can be provided on a pattern and as ingest parameter. When no timezone is provided on a pattern it is defaulted to UTC or an ingest parameter (if provided).

When timezone is not present in a pattern, but an ingest timezone parameter is provided - a date should be parsed as if it was in that timezone.
Example: pattern "uuuu-MM-dd'T'HH:mm", text "2020-01-01T01:00", timezone: -01:00 should return 2020-01-01T01:00:00-01:00
If the pattern has a timezone and a timezone parameter is provided - a date should parsed with a timezone from input, and then "recalculated" to an ingest parameter
Example "uuuu-MM-dd'T'HH:mm XXX", text "2020-01-01T01:00 -02:00", timezone -01:00 should return 2020-01-01T02:00:00-01:00
relates #38407
relates #51215
closes #63458

backports #63876

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java

@@ -78,14 +78,10 @@ private long parseMillis(String date) {
 
         @Override
         Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale locale) {
-            boolean isUtc = ZoneOffset.UTC.equals(zoneId);
 
             DateFormatter dateFormatter = DateFormatter.forPattern(format)
                 .withLocale(locale);
-            // if UTC zone is set here, the time zone specified in the format will be ignored, leading to wrong dates
-            if (isUtc == false) {
-                dateFormatter = dateFormatter.withZone(zoneId);
-            }
+
             final DateFormatter formatter = dateFormatter;
             return text -> {
                 TemporalAccessor accessor = formatter.parse(text);
@@ -105,11 +101,9 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
                     accessor = newTime.withZoneSameLocal(zoneId);
                 }
 
-                if (isUtc) {
-                    return DateFormatters.from(accessor, locale).withZoneSameInstant(ZoneOffset.UTC);
-                } else {
-                    return DateFormatters.from(accessor, locale);
-                }
+                return DateFormatters.from(accessor, locale, zoneId)
+                    .withZoneSameInstant(zoneId);
+
             };
         }
     };

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java

@@ -85,14 +85,52 @@ public void testParseWeekBasedWithLocale() {
         assumeFalse("won't work in jdk8 " +
                 "because SPI mechanism is not looking at classpath - needs ISOCalendarDataProvider in jre's ext/libs",
             JavaVersion.current().equals(JavaVersion.parse("8")));
-        String format = randomFrom("YYYY-ww");
+        String format = "YYYY-ww";
         ZoneId timezone = DateUtils.of("Europe/Amsterdam");
         Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.US);
         ZonedDateTime dateTime = javaFunction.apply("2020-33");
         //33rd week of 2020 starts on 9th August 2020 as per US locale
         assertThat(dateTime, equalTo(ZonedDateTime.of(2020,8,9,0,0,0,0,timezone)));
     }
 
+    public void testNoTimezoneOnPatternAndOverride() {
+        {
+            String format = "yyyy-MM-dd'T'HH:mm";
+            ZoneId timezone = ZoneId.of("UTC");
+            Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT);
+            // this means that hour will be 01:00 at UTC as timezone was not on a pattern, but provided as an ingest param
+            ZonedDateTime dateTime = javaFunction.apply("2020-01-01T01:00");
+            assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 01, 0, 0, 0, timezone)));
+        }
+        {
+            String format = "yyyy-MM-dd'T'HH:mm";
+            ZoneId timezone = ZoneId.of("-01:00");
+            Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT);
+            // this means that hour will be 01:00 at -01:00 as timezone was not on a pattern, but provided as an ingest param
+            ZonedDateTime dateTime = javaFunction.apply("2020-01-01T01:00");
+            assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 01, 0, 0, 0, timezone)));
+        }
+    }
+
+    public void testTimezoneOnAPatternAndNonUTCOverride() {
+        String format = "yyyy-MM-dd'T'HH:mm XXX";
+        ZoneId timezone = ZoneId.of("-01:00");
+        Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT);
+        // this means that hour will be 01:00 at -02:00 as timezone on a pattern. Converted to -01:00 as requested on ingest param
+
+        ZonedDateTime dateTime = javaFunction.apply("2020-01-01T01:00 -02:00");
+        assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 02, 0, 0, 0, timezone)));
+    }
+
+    public void testDefaultHourDefaultedToTimezoneOverride() {
+        String format = "yyyy-MM-dd";
+        ZoneId timezone = ZoneId.of("-01:00");
+        Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT);
+        // this means that hour will be 00:00 (default) at -01:00 as timezone was not on a pattern, but -01:00 was an ingest param
+        ZonedDateTime dateTime = javaFunction.apply("2020-01-01");
+        assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 0, 0, 0, 0, timezone)));
+    }
+
     public void testParseUnixMs() {
         assertThat(DateFormat.UnixMs.getFunction(null, ZoneOffset.UTC, null).apply("1000500").toInstant().toEpochMilli(),
             equalTo(1000500L));