From e9b1a22f2cee80c98e0005cd827958f357379926 Mon Sep 17 00:00:00 2001 From: cc Date: Mon, 13 Jul 2015 09:32:07 +0800 Subject: [PATCH] clean up validation of eviction plan after meta data manager view is added fix javadoc --- .../block/BlockMetadataManagerView.java | 6 - .../block/evictor/EvictionDirCandidates.java | 3 +- .../worker/block/evictor/EvictorUtils.java | 116 ------------------ .../worker/block/evictor/LRUEvictor.java | 26 +--- .../block/evictor/EvictorTestUtils.java | 97 ++++++++++++++- .../worker/block/evictor/LRUEvictorTest.java | 4 +- 6 files changed, 97 insertions(+), 155 deletions(-) delete mode 100644 servers/src/main/java/tachyon/worker/block/evictor/EvictorUtils.java diff --git a/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java b/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java index a4540e5ade65..d0e6ab039a2c 100644 --- a/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java +++ b/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java @@ -73,12 +73,6 @@ public BlockMetadataManagerView(BlockMetadataManager manager, Set pinne } } - // TODO: this is only used for {@link EvictorUtils#validCascadingPlan}, remove this when - // validCascadingPlan is removed from LRUEvictor - public BlockMetadataManager getMetadataManager() { - return mMetadataManager; - } - /** * Test if the block is pinned. * diff --git a/servers/src/main/java/tachyon/worker/block/evictor/EvictionDirCandidates.java b/servers/src/main/java/tachyon/worker/block/evictor/EvictionDirCandidates.java index 561b4dd7e713..d4958ee3601d 100644 --- a/servers/src/main/java/tachyon/worker/block/evictor/EvictionDirCandidates.java +++ b/servers/src/main/java/tachyon/worker/block/evictor/EvictionDirCandidates.java @@ -31,8 +31,7 @@ * total bytes of added blocks. Assume meta data of StorageDir will not be changed during adding * blocks. * - * Example usage can be found in - * {@link LRUEvictor#freeSpace(long, tachyon.worker.block.BlockStoreLocation)}. + * Example usage can be found in {@link LRUEvictor#freeSpaceWithView}. */ class EvictionDirCandidates { /** Map from StorageDirView to pair of list of candidate blockIds and their total size in bytes */ diff --git a/servers/src/main/java/tachyon/worker/block/evictor/EvictorUtils.java b/servers/src/main/java/tachyon/worker/block/evictor/EvictorUtils.java deleted file mode 100644 index cf20f742d886..000000000000 --- a/servers/src/main/java/tachyon/worker/block/evictor/EvictorUtils.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the University of California, Berkeley under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package tachyon.worker.block.evictor; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import tachyon.Pair; -import tachyon.worker.block.BlockMetadataManager; -import tachyon.worker.block.BlockStoreLocation; -import tachyon.worker.block.meta.BlockMeta; -import tachyon.worker.block.meta.StorageDir; - -public class EvictorUtils { - /** - * Checks whether the plan of a cascading evictor is valid. - * - * A cascading evictor will try to free space by recursively moving blocks to next 1 tier and - * evict blocks only in the bottom tier. - * - * The plan is invalid when the requested space can not be satisfied or lower level of tiers do - * not have enough space to hold blocks moved from higher level of tiers. - * - * @param bytesToBeAvailable requested bytes to be available after eviction - * @param plan the eviction plan, should not be empty - * @param metaManager the meta data manager - * @return true if the above requirements are satisfied, otherwise false. - */ - // TODO: unit test this method - public static boolean validCascadingPlan(long bytesToBeAvailable, EvictionPlan plan, - BlockMetadataManager metaManager) throws IOException { - // reassure the plan is feasible: enough free space to satisfy bytesToBeAvailable, and enough - // space in lower tier to move blocks in upper tier there - - // Map from dir to a pair of bytes to be available in this dir and bytes to move into this dir - // after the plan taking action - Map> spaceInfoInDir = new HashMap>(); - - for (long blockId : plan.toEvict()) { - BlockMeta block = metaManager.getBlockMeta(blockId); - StorageDir dir = block.getParentDir(); - if (spaceInfoInDir.containsKey(dir)) { - Pair spaceInfo = spaceInfoInDir.get(dir); - spaceInfo.setFirst(spaceInfo.getFirst() + block.getBlockSize()); - } else { - spaceInfoInDir.put(dir, new Pair( - dir.getAvailableBytes() + block.getBlockSize(), 0L)); - } - } - - for (Pair move : plan.toMove()) { - long blockId = move.getFirst(); - BlockMeta block = metaManager.getBlockMeta(blockId); - long blockSize = block.getBlockSize(); - StorageDir srcDir = block.getParentDir(); - StorageDir destDir = metaManager.getDir(move.getSecond()); - - if (spaceInfoInDir.containsKey(srcDir)) { - Pair spaceInfo = spaceInfoInDir.get(srcDir); - spaceInfo.setFirst(spaceInfo.getFirst() + blockSize); - } else { - spaceInfoInDir - .put(srcDir, new Pair(srcDir.getAvailableBytes() + blockSize, 0L)); - } - - if (spaceInfoInDir.containsKey(destDir)) { - Pair spaceInfo = spaceInfoInDir.get(destDir); - spaceInfo.setSecond(spaceInfo.getSecond() + blockSize); - } else { - spaceInfoInDir.put(destDir, new Pair(destDir.getAvailableBytes(), blockSize)); - } - } - - // the top tier among all tiers where blocks in the plan reside in - int topTierAlias = Integer.MAX_VALUE; - for (StorageDir dir : spaceInfoInDir.keySet()) { - topTierAlias = Math.min(topTierAlias, dir.getParentTier().getTierAlias()); - } - long maxSpace = Long.MIN_VALUE; // maximum bytes to be available in a dir in the top tier - for (StorageDir dir : spaceInfoInDir.keySet()) { - if (dir.getParentTier().getTierAlias() == topTierAlias) { - Pair space = spaceInfoInDir.get(dir); - maxSpace = Math.max(maxSpace, space.getFirst() - space.getSecond()); - } - } - if (maxSpace < bytesToBeAvailable) { - // plan is invalid because requested space can not be satisfied in the top tier - return false; - } - - for (StorageDir dir : spaceInfoInDir.keySet()) { - Pair spaceInfo = spaceInfoInDir.get(dir); - if (spaceInfo.getFirst() < spaceInfo.getSecond()) { - // plan is invalid because there is not enough space in this dir to hold the blocks waiting - // to be moved into this dir - return false; - } - } - - return true; - } -} diff --git a/servers/src/main/java/tachyon/worker/block/evictor/LRUEvictor.java b/servers/src/main/java/tachyon/worker/block/evictor/LRUEvictor.java index 2c37a9334f78..7cec664e3048 100644 --- a/servers/src/main/java/tachyon/worker/block/evictor/LRUEvictor.java +++ b/servers/src/main/java/tachyon/worker/block/evictor/LRUEvictor.java @@ -180,7 +180,6 @@ private StorageDirView cascadingEvict(long bytesToBeAvailable, BlockStoreLocatio return candidateDirView; } - @Override public EvictionPlan freeSpaceWithView(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataManagerView view) throws IOException { @@ -194,31 +193,8 @@ public EvictionPlan freeSpaceWithView(long bytesToBeAvailable, BlockStoreLocatio if (candidateDir == null) { return null; } - if (plan.isEmpty()) { - return plan; - } - - // TODO: remove this check after implementation of metadata view - // assure all blocks are in the store, if not, remove from plan and lru cache - Iterator> moveIt = plan.toMove().iterator(); - while (moveIt.hasNext()) { - long id = moveIt.next().getFirst(); - if (null == mManagerView.getBlockMeta(id)) { - mLRUCache.remove(id); - moveIt.remove(); - } - } - Iterator evictIt = plan.toEvict().iterator(); - while (evictIt.hasNext()) { - long id = evictIt.next(); - if (null == mManagerView.getBlockMeta(id)) { - mLRUCache.remove(id); - evictIt.remove(); - } - } - return EvictorUtils.validCascadingPlan(bytesToBeAvailable, plan, - mManagerView.getMetadataManager()) ? plan : null; + return plan; } @Override diff --git a/servers/src/test/java/tachyon/worker/block/evictor/EvictorTestUtils.java b/servers/src/test/java/tachyon/worker/block/evictor/EvictorTestUtils.java index a7ec33b63acc..4ec7f5f23c49 100644 --- a/servers/src/test/java/tachyon/worker/block/evictor/EvictorTestUtils.java +++ b/servers/src/test/java/tachyon/worker/block/evictor/EvictorTestUtils.java @@ -18,8 +18,9 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.junit.Assert; @@ -36,11 +37,11 @@ import tachyon.conf.TachyonConf; import tachyon.util.CommonUtils; import tachyon.worker.block.BlockMetadataManager; -import tachyon.worker.block.BlockMetadataManagerView; import tachyon.worker.block.BlockStoreEventListener; import tachyon.worker.block.BlockStoreLocation; import tachyon.worker.block.io.BlockWriter; import tachyon.worker.block.io.LocalFileBlockWriter; +import tachyon.worker.block.meta.BlockMeta; import tachyon.worker.block.meta.StorageDir; import tachyon.worker.block.meta.TempBlockMeta; @@ -267,7 +268,95 @@ public static boolean validNonCascadingPlan(long bytesToBeAvailable, EvictionPla } /** - * Only when plan is not null and at least one of {@link EvictorUtils#validCascadingPlan}, + * Checks whether the plan of a cascading evictor is valid. + * + * A cascading evictor will try to free space by recursively moving blocks to next 1 tier and + * evict blocks only in the bottom tier. + * + * The plan is invalid when the requested space can not be satisfied or lower level of tiers do + * not have enough space to hold blocks moved from higher level of tiers. + * + * @param bytesToBeAvailable requested bytes to be available after eviction + * @param plan the eviction plan, should not be empty + * @param metaManager the meta data manager + * @return true if the above requirements are satisfied, otherwise false. + */ + // TODO: unit test this method + public static boolean validCascadingPlan(long bytesToBeAvailable, EvictionPlan plan, + BlockMetadataManager metaManager) throws IOException { + // reassure the plan is feasible: enough free space to satisfy bytesToBeAvailable, and enough + // space in lower tier to move blocks in upper tier there + + // Map from dir to a pair of bytes to be available in this dir and bytes to move into this dir + // after the plan taking action + Map> spaceInfoInDir = new HashMap>(); + + for (long blockId : plan.toEvict()) { + BlockMeta block = metaManager.getBlockMeta(blockId); + StorageDir dir = block.getParentDir(); + if (spaceInfoInDir.containsKey(dir)) { + Pair spaceInfo = spaceInfoInDir.get(dir); + spaceInfo.setFirst(spaceInfo.getFirst() + block.getBlockSize()); + } else { + spaceInfoInDir.put(dir, new Pair( + dir.getAvailableBytes() + block.getBlockSize(), 0L)); + } + } + + for (Pair move : plan.toMove()) { + long blockId = move.getFirst(); + BlockMeta block = metaManager.getBlockMeta(blockId); + long blockSize = block.getBlockSize(); + StorageDir srcDir = block.getParentDir(); + StorageDir destDir = metaManager.getDir(move.getSecond()); + + if (spaceInfoInDir.containsKey(srcDir)) { + Pair spaceInfo = spaceInfoInDir.get(srcDir); + spaceInfo.setFirst(spaceInfo.getFirst() + blockSize); + } else { + spaceInfoInDir + .put(srcDir, new Pair(srcDir.getAvailableBytes() + blockSize, 0L)); + } + + if (spaceInfoInDir.containsKey(destDir)) { + Pair spaceInfo = spaceInfoInDir.get(destDir); + spaceInfo.setSecond(spaceInfo.getSecond() + blockSize); + } else { + spaceInfoInDir.put(destDir, new Pair(destDir.getAvailableBytes(), blockSize)); + } + } + + // the top tier among all tiers where blocks in the plan reside in + int topTierAlias = Integer.MAX_VALUE; + for (StorageDir dir : spaceInfoInDir.keySet()) { + topTierAlias = Math.min(topTierAlias, dir.getParentTier().getTierAlias()); + } + long maxSpace = Long.MIN_VALUE; // maximum bytes to be available in a dir in the top tier + for (StorageDir dir : spaceInfoInDir.keySet()) { + if (dir.getParentTier().getTierAlias() == topTierAlias) { + Pair space = spaceInfoInDir.get(dir); + maxSpace = Math.max(maxSpace, space.getFirst() - space.getSecond()); + } + } + if (maxSpace < bytesToBeAvailable) { + // plan is invalid because requested space can not be satisfied in the top tier + return false; + } + + for (StorageDir dir : spaceInfoInDir.keySet()) { + Pair spaceInfo = spaceInfoInDir.get(dir); + if (spaceInfo.getFirst() < spaceInfo.getSecond()) { + // plan is invalid because there is not enough space in this dir to hold the blocks waiting + // to be moved into this dir + return false; + } + } + + return true; + } + + /** + * Only when plan is not null and at least one of {@link EvictorTestUtils#validCascadingPlan}, * {@link #validNonCascadingPlan} is true, the assertion will be passed, used in unit test. * * @param bytesToBeAvailable the requested bytes to be available @@ -279,6 +368,6 @@ public static void assertValidPlan(long bytesToBeAvailable, EvictionPlan plan, BlockMetadataManager metaManager) throws IOException { Assert.assertNotNull(plan); Assert.assertTrue(validNonCascadingPlan(bytesToBeAvailable, plan, metaManager) - || EvictorUtils.validCascadingPlan(bytesToBeAvailable, plan, metaManager)); + || EvictorTestUtils.validCascadingPlan(bytesToBeAvailable, plan, metaManager)); } } diff --git a/servers/src/test/java/tachyon/worker/block/evictor/LRUEvictorTest.java b/servers/src/test/java/tachyon/worker/block/evictor/LRUEvictorTest.java index 5162c813bd31..41063530259e 100644 --- a/servers/src/test/java/tachyon/worker/block/evictor/LRUEvictorTest.java +++ b/servers/src/test/java/tachyon/worker/block/evictor/LRUEvictorTest.java @@ -109,7 +109,7 @@ public void cascadingEvictionTest1() throws IOException { for (int i = 0; i < nDir; i ++) { EvictionPlan plan = mEvictor.freeSpaceWithView(smallestCapacity, anyDirInFirstTier, mManagerView); - Assert.assertTrue(EvictorUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager)); + Assert.assertTrue(EvictorTestUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager)); Assert.assertEquals(0, plan.toEvict().size()); Assert.assertEquals(1, plan.toMove().size()); long blockId = plan.toMove().get(0).getFirst(); @@ -141,7 +141,7 @@ public void cascadingEvictionTest2() throws IOException { for (int i = 0; i < nDirInFirstTier; i ++) { EvictionPlan plan = mEvictor.freeSpaceWithView(smallestCapacity, anyDirInFirstTier, mManagerView); - Assert.assertTrue(EvictorUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager)); + Assert.assertTrue(EvictorTestUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager)); // least recently used block in the first tier needs to be moved to the second tier Assert.assertEquals(1, plan.toMove().size()); long blockIdMovedInFirstTier = plan.toMove().get(0).getFirst();