From 508a604144ac2b5511dfb82795567656c38846c7 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Thu, 15 Oct 2020 12:13:30 +0530 Subject: [PATCH 1/4] HADOOP-17306. RawLocalFileSystem's lastModifiedTime() looses milli seconds in JDK < 10.b09 --- .../apache/hadoop/fs/RawLocalFileSystem.java | 9 +++++++-- .../fs/TestRawLocalFileSystemContract.java | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 72eeb99a4ea5d..257efd44ce0a0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -72,7 +72,12 @@ public class RawLocalFileSystem extends FileSystem { public static void useStatIfAvailable() { useDeprecatedFileStatus = !Stat.isAvailable(); } - + + @VisibleForTesting + static void setUseDeprecatedFileStatus(boolean useDeprecatedFileStatus) { + RawLocalFileSystem.useDeprecatedFileStatus = useDeprecatedFileStatus; + } + public RawLocalFileSystem() { workingDir = getInitialWorkingDirectory(); } @@ -700,7 +705,7 @@ private static long getLastAccessTime(File f) throws IOException { DeprecatedRawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) throws IOException { super(f.length(), f.isDirectory(), 1, defaultBlockSize, - f.lastModified(), getLastAccessTime(f), + Files.getLastModifiedTime(f.toPath()).toMillis(), getLastAccessTime(f), null, null, null, new Path(f.getPath()).makeQualified(fs.getUri(), fs.getWorkingDirectory())); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java index b51419d8c53f9..7f8ee771e4557 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java @@ -203,4 +203,24 @@ public void testPermission() throws Exception { } } + @Test + public void testMTimeAtime() throws IOException { + RawLocalFileSystem.setUseDeprecatedFileStatus(true); + try { + Path testDir = getTestBaseDir(); + String testFilename = "testmtime"; + Path path = new Path(testDir, testFilename); + Path file = new Path(path, "file"); + fs.create(file); + long now = System.currentTimeMillis(); + long mtime = (now % 1000 == 0) ? now + 1 : now; + long atime = (now % 1000 == 0) ? now + 2 : now; + fs.setTimes(file, mtime, atime); + FileStatus fileStatus = fs.getFileStatus(file); + assertEquals(mtime, fileStatus.getModificationTime()); + assertEquals(atime, fileStatus.getAccessTime()); + } finally { + RawLocalFileSystem.useStatIfAvailable(); + } + } } From af1d2cc5ff8c7c0d7171d40016df19d325f3cc1c Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Thu, 15 Oct 2020 17:54:54 +0530 Subject: [PATCH 2/4] Exclude assertion for MAC --- .../org/apache/hadoop/fs/TestRawLocalFileSystemContract.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java index 7f8ee771e4557..2cd72bbd3074d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java @@ -217,7 +217,10 @@ public void testMTimeAtime() throws IOException { long atime = (now % 1000 == 0) ? now + 2 : now; fs.setTimes(file, mtime, atime); FileStatus fileStatus = fs.getFileStatus(file); - assertEquals(mtime, fileStatus.getModificationTime()); + if (!Shell.MAC) { + //HADOOP-17306 ; JDK-8177809 not fixed in MacOs + assertEquals(mtime, fileStatus.getModificationTime()); + } assertEquals(atime, fileStatus.getAccessTime()); } finally { RawLocalFileSystem.useStatIfAvailable(); From 5d8c456f294edc5d6b22f05ad77f6f31215b9da8 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Mon, 19 Oct 2020 11:04:44 +0530 Subject: [PATCH 3/4] Updated skip comment for MacOs --- .../org/apache/hadoop/fs/TestRawLocalFileSystemContract.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java index 2cd72bbd3074d..cb45c9ef81549 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java @@ -218,7 +218,8 @@ public void testMTimeAtime() throws IOException { fs.setTimes(file, mtime, atime); FileStatus fileStatus = fs.getFileStatus(file); if (!Shell.MAC) { - //HADOOP-17306 ; JDK-8177809 not fixed in MacOs + // HADOOP-17306 ; Skip MacOS because HFS+ does not support + // milliseconds for mtime. assertEquals(mtime, fileStatus.getModificationTime()); } assertEquals(atime, fileStatus.getAccessTime()); From 4e558115fe0a4da56f9bb105eb78fc73baf644d2 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Mon, 19 Oct 2020 11:11:16 +0530 Subject: [PATCH 4/4] Checkstyle fix. --- .../main/java/org/apache/hadoop/fs/RawLocalFileSystem.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 257efd44ce0a0..129409663c9ed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -705,8 +705,8 @@ private static long getLastAccessTime(File f) throws IOException { DeprecatedRawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) throws IOException { super(f.length(), f.isDirectory(), 1, defaultBlockSize, - Files.getLastModifiedTime(f.toPath()).toMillis(), getLastAccessTime(f), - null, null, null, + Files.getLastModifiedTime(f.toPath()).toMillis(), + getLastAccessTime(f),null, null, null, new Path(f.getPath()).makeQualified(fs.getUri(), fs.getWorkingDirectory())); }