From 0adac721e3de6eacec97967d19b466b405f9427a Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sun, 4 Jun 2023 22:15:54 -0700 Subject: [PATCH 1/3] HADOOP-18756. S3A prefetch - CachingBlockManager to use AtomicBoolean for closed flag --- .../prefetch/SingleFilePerBlockCache.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java index d58a695ec2399..4e1c3faab635a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.slf4j.Logger; @@ -66,7 +67,7 @@ public class SingleFilePerBlockCache implements BlockCache { */ private int numGets = 0; - private boolean closed; + private final AtomicBoolean closed; private final PrefetchingStatistics prefetchingStatistics; @@ -108,6 +109,7 @@ public String toString() { */ public SingleFilePerBlockCache(PrefetchingStatistics prefetchingStatistics) { this.prefetchingStatistics = requireNonNull(prefetchingStatistics); + this.closed = new AtomicBoolean(false); } /** @@ -141,7 +143,7 @@ public int size() { */ @Override public void get(int blockNumber, ByteBuffer buffer) throws IOException { - if (closed) { + if (closed.get()) { return; } @@ -192,7 +194,7 @@ private Entry getEntry(int blockNumber) { @Override public void put(int blockNumber, ByteBuffer buffer, Configuration conf, LocalDirAllocator localDirAllocator) throws IOException { - if (closed) { + if (closed.get()) { return; } @@ -258,27 +260,23 @@ protected Path getCacheFilePath(final Configuration conf, @Override public void close() throws IOException { - if (closed) { - return; - } - - closed = true; - - LOG.info(getStats()); - int numFilesDeleted = 0; - - for (Entry entry : blocks.values()) { - try { - Files.deleteIfExists(entry.path); - prefetchingStatistics.blockRemovedFromFileCache(); - numFilesDeleted++; - } catch (IOException e) { - LOG.debug("Failed to delete cache file {}", entry.path, e); + if (closed.compareAndSet(false, true)) { + LOG.info(getStats()); + int numFilesDeleted = 0; + + for (Entry entry : blocks.values()) { + try { + Files.deleteIfExists(entry.path); + prefetchingStatistics.blockRemovedFromFileCache(); + numFilesDeleted++; + } catch (IOException e) { + LOG.debug("Failed to delete cache file {}", entry.path, e); + } } - } - if (numFilesDeleted > 0) { - LOG.info("Deleted {} cache files", numFilesDeleted); + if (numFilesDeleted > 0) { + LOG.info("Deleted {} cache files", numFilesDeleted); + } } } From 7f48042efed52119a991381a1cfc196bb98ddd38 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 8 Jun 2023 10:37:21 -0700 Subject: [PATCH 2/3] addendum --- .../hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java index 8f1fb43920403..a732aff30934e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java @@ -336,7 +336,7 @@ protected Path getCacheFilePath(final Configuration conf, @Override public void close() throws IOException { if (closed.compareAndSet(false, true)) { - LOG.info(getStats()); + LOG.debug(getStats()); int numFilesDeleted = 0; for (Entry entry : blocks.values()) { @@ -359,9 +359,7 @@ public void close() throws IOException { } } - if (numFilesDeleted > 0) { - LOG.info("Deleted {} cache files", numFilesDeleted); - } + LOG.debug("Prefetch cache close: Deleted {} cache files", numFilesDeleted); } } From 325a03094f89826ece2420b921693f626640419f Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 12 Jun 2023 08:31:09 -0700 Subject: [PATCH 3/3] addendum --- .../apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java index a732aff30934e..e043fbd904be8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java @@ -353,7 +353,7 @@ public void close() throws IOException { prefetchingStatistics.blockRemovedFromFileCache(); numFilesDeleted++; } catch (IOException e) { - LOG.error("Failed to delete cache file {}", entry.path, e); + LOG.warn("Failed to delete cache file {}", entry.path, e); } finally { entry.releaseLock(Entry.LockType.WRITE); }