Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
FALCON-2023 Feed eviction fails when feed locations stats and meta …
…does not have time pattern

Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>

Reviewers: " sandeepSamudrala <sandysmdl@gmail.com>, Ying Zheng <yzheng@hortonworks.com>, Balu Vellanki <balu@apache.org>"

Closes #182 from vramachan/FALCON-2023
  • Loading branch information
vramachan authored and bvellanki committed Jun 13, 2016
1 parent af7407f commit df6b53c83520c9cd015dfbd0d752f25d18cd9c3f
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 2 deletions.
@@ -358,13 +358,20 @@ private List<Path> discoverInstanceToDelete(String inPath, TimeZone timeZone, Da

private FileStatus[] findFilesForFeed(FileSystem fs, String feedBasePath) throws IOException {
Matcher matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
boolean regexMatchFound = false;
while (matcher.find()) {
regexMatchFound = true;
String var = feedBasePath.substring(matcher.start(), matcher.end());
feedBasePath = feedBasePath.replaceAll(Pattern.quote(var), "*");
matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
}
LOG.info("Searching for {}", feedBasePath);
return fs.globStatus(new Path(feedBasePath));
if (regexMatchFound) {
LOG.info("Searching for {}", feedBasePath);
return fs.globStatus(new Path(feedBasePath));
} else {
LOG.info("Ignoring static path {}", feedBasePath);
return null;
}
}

private boolean isDateInRange(Date date, Date start) {
@@ -462,6 +462,43 @@ public void testFeedBasePathExists() throws Exception {
}
}

@Test
public void testEvictionStatsMetaWithNoPattern() throws Exception {
try {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
fs.delete(new Path("/"), true);
stream.clear();

Pair<List<String>, List<String>> pair = createTestData("/data");
createDir("/stats");
createDir("/meta");
createTestData("/tmp");
final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
FeedEvictor.main(new String[] {
"-feedBasePath",
getFeedBasePath(LocationType.DATA, storageUrl) + "#"
+ getStatsOrMetaPath(LocationType.STATS, storageUrl)
+ "#" + getStatsOrMetaPath(LocationType.META, storageUrl)
+ "#" + getFeedBasePath(LocationType.TMP, storageUrl),
"-retentionType", "instance",
"-retentionLimit", "months(5)",
"-timeZone", "UTC",
"-frequency", "hourly",
"-logFile", conf.get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
+ "/falcon/staging/feed/2012-01-01-04-00", "-falconFeedStorageType",
Storage.TYPE.FILESYSTEM.name(),
});

// should not throw exception
// stats and meta dir should not be deleted
Assert.assertTrue(isDirPresent("/stats"));
Assert.assertTrue(isDirPresent("/meta"));
} catch (Exception e) {
Assert.fail("Unknown exception", e);
}
}


private Pair<List<String>, List<String>> createTestData(String locationType) throws Exception {
Configuration conf = cluster.getConf();
@@ -482,6 +519,12 @@ private Pair<List<String>, List<String>> createTestData(String locationType) thr
return Pair.of(inRange, outOfRange);
}

private void createDir(String locationType) throws Exception {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
touch(fs, locationType, false);
}

private Pair<List<String>, List<String>> createTestData(String feed, String mask,
int period, TimeUnit timeUnit,
String locationType) throws Exception {
@@ -542,11 +585,21 @@ private void touch(FileSystem fs, String path, boolean generateFiles) throws Exc
}
}

private boolean isDirPresent(String path) throws Exception {
FileSystem fs = FileSystem.get(cluster.getConf());
return fs.exists(new Path(path));
}

private String getFeedBasePath(LocationType locationType, String storageUrl) {
return locationType.name() + "=" + storageUrl
+ "/" + locationType.name().toLowerCase() + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}";
}

private String getStatsOrMetaPath(LocationType locationType, String storageUrl) {
return locationType.name() + "=" + storageUrl
+ "/" + locationType.name().toLowerCase();
}

private static class InMemoryWriter extends PrintStream {

private final StringBuffer buffer = new StringBuffer();

0 comments on commit df6b53c

Please sign in to comment.