From c67ca8ab8c798f9674b112a3ce94fc0c671e7d3d Mon Sep 17 00:00:00 2001 From: Gil Cottle Date: Thu, 23 Mar 2023 12:54:52 -0400 Subject: [PATCH 1/3] HADOOP-18675: Fix CachedSASToken noisy log errors when SAS token has YYYY-MM-DD expiration --- .../fs/azurebfs/utils/CachedSASToken.java | 29 +++++++++++++++++-- .../fs/azurebfs/utils/TestCachedSASToken.java | 25 ++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java index 559c31dedd1a7..7541ca9d7e01a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java @@ -20,10 +20,16 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; +import java.time.chrono.ChronoLocalDate; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; @@ -39,6 +45,13 @@ */ public final class CachedSASToken { public static final Logger LOG = LoggerFactory.getLogger(CachedSASToken.class); + + private static final DateTimeFormatter isoDateMidnight = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_DATE) + .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) + .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) + .toFormatter(); + private final long minExpirationInSeconds; private String sasToken; private OffsetDateTime sasExpiry; @@ -114,8 +127,20 @@ private static OffsetDateTime getExpiry(String token) { OffsetDateTime seDate = OffsetDateTime.MIN; try { seDate = OffsetDateTime.parse(seValue, DateTimeFormatter.ISO_DATE_TIME); - } catch (DateTimeParseException ex) { - LOG.error("Error parsing se query parameter ({}) from SAS.", seValue, ex); + } catch (DateTimeParseException dateTimeException) { + try { + TemporalAccessor dt = isoDateMidnight.parseBest(seValue, OffsetDateTime::from, LocalDateTime::from); + if (dt instanceof OffsetDateTime) { + seDate = (OffsetDateTime) dt; + } else if (dt instanceof LocalDateTime) { + seDate = ((LocalDateTime) dt).atOffset(ZoneOffset.UTC); + } else { + throw dateTimeException; + } + } catch (DateTimeParseException dateOnlyException) { + // log original exception + LOG.error("Error parsing se query parameter ({}) from SAS as ISO_DATE_TIME or ISO_DATE.", seValue, dateTimeException); + } } String signedKeyExpiry = "ske="; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestCachedSASToken.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestCachedSASToken.java index cbba80877206f..08602f440d0bd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestCachedSASToken.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestCachedSASToken.java @@ -78,6 +78,31 @@ public void testUpdateAndGet() throws IOException { Assert.assertTrue(token3 == cachedToken); } + @Test + public void testValidExpirationParsing() { + CachedSASToken cachedSasToken = new CachedSASToken(); + String[] values = { + "2123-03-24T00:06:46Z", // sample timestamp from azure portal + "2124-03-30", // sample YYYY-MM-DD date format generated from az cli + "2125-03-30Z", // sample YYYY-MM-DD[offset] date format + }; + + for (String se : values) { + cachedSasToken.setForTesting(null, null); + String token = "se=" + se; + + // set first time and ensure reference equality + cachedSasToken.update(token); + String cachedToken = cachedSasToken.get(); + Assert.assertTrue(token == cachedToken); + + // update with same token and ensure reference equality + cachedSasToken.update(token); + cachedToken = cachedSasToken.get(); + Assert.assertTrue(token == cachedToken); + } + } + @Test public void testGetExpiration() throws IOException { CachedSASToken cachedSasToken = new CachedSASToken(); From 77444f6f02d3eddc393d14fb96cff7f8525ebfa0 Mon Sep 17 00:00:00 2001 From: Gil Cottle Date: Thu, 23 Mar 2023 14:04:13 -0400 Subject: [PATCH 2/3] remove unused imports --- .../org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java index 7541ca9d7e01a..85497af993005 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java @@ -20,11 +20,9 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.time.LocalDate; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; -import java.time.chrono.ChronoLocalDate; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; From ac12f4be11773c4e6290cae43124149f9683bb75 Mon Sep 17 00:00:00 2001 From: Gil Cottle Date: Thu, 23 Mar 2023 14:58:02 -0400 Subject: [PATCH 3/3] checkstyle --- .../org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java index 85497af993005..3b059c9878054 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/CachedSASToken.java @@ -44,7 +44,7 @@ public final class CachedSASToken { public static final Logger LOG = LoggerFactory.getLogger(CachedSASToken.class); - private static final DateTimeFormatter isoDateMidnight = new DateTimeFormatterBuilder() + private static final DateTimeFormatter ISO_DATE_MIDNIGHT = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ISO_DATE) .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) @@ -127,7 +127,7 @@ private static OffsetDateTime getExpiry(String token) { seDate = OffsetDateTime.parse(seValue, DateTimeFormatter.ISO_DATE_TIME); } catch (DateTimeParseException dateTimeException) { try { - TemporalAccessor dt = isoDateMidnight.parseBest(seValue, OffsetDateTime::from, LocalDateTime::from); + TemporalAccessor dt = ISO_DATE_MIDNIGHT.parseBest(seValue, OffsetDateTime::from, LocalDateTime::from); if (dt instanceof OffsetDateTime) { seDate = (OffsetDateTime) dt; } else if (dt instanceof LocalDateTime) {