Skip to content
Permalink
Browse files
provide means to detect and return s3 prefixes as directory resources
  • Loading branch information
mbenson committed Mar 10, 2022
1 parent 6ba67c0 commit 7f3da56d4a123c9921bf2e49578869dcf87fbb73
Showing 3 changed files with 65 additions and 12 deletions.
@@ -157,6 +157,12 @@ public ObjectResource(Project project) {
this.versionId = versionId;
}

static ObjectResource ofPrefix(Project project, S3Client s3, String bucket, String prefix) {
ObjectResource result = new ObjectResource(project,s3,bucket,prefix);
result._setDirectory(true);
return result;
}

/**
* Create a new {@link ObjectResource} fully-formed.
*
@@ -701,4 +707,9 @@ private Optional<HeadObjectResponse> head() {
private S3Client s3() {
return ProjectUtils.requireComponent(getProject(), s3, Client.class);
}

private void _setDirectory(boolean directory) {
super.setDirectory(directory);
}
}

@@ -49,6 +49,7 @@
* {@link ResourceCollection} of {@link ObjectResource}s.
*/
public class ObjectResources extends S3DataType implements ResourceCollection {

/**
* Default delimiter.
*/
@@ -70,6 +71,7 @@ private static <T> Stream<T> sequentialStream(Iterator<T> iterator) {

private volatile Optional<ResourceCollection> resourceCache;
private volatile boolean cache = true;
private volatile boolean includePrefixes;

/**
* Create a new {@link ObjectResources} instance.
@@ -138,7 +140,7 @@ public Iterator<Resource> iterator() {
return resourceCache.get().iterator();
}
final S3Finder finder = new S3Finder(getProject(), s3(), require(getBucket(), "@bucket"), as,
require(getDelimiter(), "@delimiter"), patterns, isCaseSensitive());
require(getDelimiter(), "@delimiter"), patterns, isCaseSensitive(), isIncludePrefixes());

final Optional<Set<ObjectResource>> cacheSet;
if (isCache()) {
@@ -324,6 +326,29 @@ public void setCache(boolean cache) {
}
}

/**
* Learn whether to include S3 object prefixes as "directory" type
* {@link Resource}s, default {@code false}.
*
* @return {@code boolean}
*/
public boolean isIncludePrefixes() {
return includePrefixes;
}

/**
* Set whether to include S3 object prefixes as "directory" type
* {@link Resource}s, default {@code false}.
* @param includePrefixes {@code boolean}
*/
public void setIncludePrefixes(boolean includePrefixes) {
checkAttributesAllowed();
if (includePrefixes != this.includePrefixes) {
this.includePrefixes = includePrefixes;
resetResourceCache();
}
}

/**
* Add a nested {@link PatternSet}.
*
@@ -25,6 +25,7 @@
import java.util.Deque;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.BooleanSupplier;
@@ -62,6 +63,10 @@ class S3Finder implements Supplier<Optional<ObjectResource>> {
static Comparator<Atom<?>> COMPARATOR = Comparator.<Atom<?>, String> comparing(Atom::key)
.thenComparing(Atom::latest).thenComparing(Atom::lastModified);

static Atom<String> asDir(S3Object o) {
return new Atom<String>(o.key(), o::key, () -> null, () -> true, o::lastModified, ObjectResource::ofPrefix);
}

static Atom<S3Object> of(S3Object o) {
return new Atom<S3Object>(o, o::key, () -> null, () -> true, o::lastModified, ObjectResource::new);
}
@@ -138,7 +143,7 @@ protected BaseFrame(S3Finder finder, RESPONSE listing, Supplier<String> prefix,

return remainingDepth > 1;
});
if (canRecurse) {
if (canRecurse || finder.includePrefixes) {
this.prefixes = prefixes.stream().filter(this::allowPrefix).iterator();
} else {
this.prefixes = Collections.emptyIterator();
@@ -179,10 +184,19 @@ public String toString() {
}
}

private static Stream<Atom<?>> atoms(ListObjectsV2Response objects, boolean includePrefixes) {
return objects.contents().stream().<Atom<?>> map(o -> {
if (o.key().equals(objects.prefix())) {
return includePrefixes ? Atom.asDir(o) : null;
}
return Atom.of(o);
}).filter(Objects::nonNull);
}

static class ObjectsFrame extends BaseFrame<ListObjectsV2Response, ObjectsFrame> {

ObjectsFrame(S3Finder finder, ListObjectsV2Response objects) {
super(finder, objects, objects::prefix, objects.commonPrefixes(), objects.contents().stream().map(Atom::of),
super(finder, objects, objects::prefix, objects.commonPrefixes(), atoms(objects, finder.includePrefixes),
r -> new ObjectsFrame(finder, r));
}

@@ -204,21 +218,21 @@ private static boolean breaksKey(ListObjectVersionsResponse versions) {
return versions.isTruncated() && versions.nextKeyMarker().equals(versions.keyMarker());
}

private static Stream<Atom<?>> atoms(ListObjectVersionsResponse versions) {
private static Stream<Atom<?>> atoms(ListObjectVersionsResponse versions, boolean includePrefixes) {
Stream<Atom<?>> result =
Stream.concat(versions.deleteMarkers().stream().map(Atom::of), versions.versions().stream().map(Atom::of));

if (!includePrefixes) {
result = result.filter(a -> !a.key().equals(versions.prefix()));
}
if (versions.isTruncated()) {
// attempt to to facilitate version ordering by omitting last/next
// key:

// attempt to facilitate version ordering by omitting last/next key:
if (!breaksKey(versions)) {
final String nextKey = versions.nextKeyMarker();
result = result.filter(atom -> !nextKey.equals(atom.key()));
}
}
// we think plain objects listing is already sorted by key, so just sort
// versions atoms here:
// we think plain objects listing is already sorted by key, so just sort versions atoms here:
result = result.sorted(Atom.COMPARATOR);

return result;
@@ -227,8 +241,8 @@ private static Stream<Atom<?>> atoms(ListObjectVersionsResponse versions) {
static class VersionsFrame extends BaseFrame<ListObjectVersionsResponse, VersionsFrame> {

VersionsFrame(S3Finder finder, ListObjectVersionsResponse versions) {
super(finder, versions, versions::prefix, versions.commonPrefixes(), atoms(versions),
r -> new VersionsFrame(finder, r));
super(finder, versions, versions::prefix, versions.commonPrefixes(),
atoms(versions, finder.includePrefixes), r -> new VersionsFrame(finder, r));
}

@Override
@@ -280,6 +294,7 @@ private static Optional<String> determinePrefix(Set<TokenizedPattern> includes)
private final String delimiter;
private final boolean caseSensitive;
private final Pair<Set<TokenizedPattern>, Set<TokenizedPattern>> patterns;
private final boolean includePrefixes;

/**
* Create a new {@link S3Finder} instance.
@@ -290,15 +305,17 @@ private static Optional<String> determinePrefix(Set<TokenizedPattern> includes)
* @param precision
* @param delimiter
* @param patterns
* @param includePrefixes
*/
S3Finder(Project project, S3Client s3, String bucket, Precision precision, String delimiter, PatternSet patterns,
boolean caseSensitive) {
boolean caseSensitive, boolean includePrefixes) {
this.project = project;
this.s3 = s3;
this.bucket = bucket;
this.delimiter = delimiter;
this.patterns = tokenize(patterns);
this.caseSensitive = caseSensitive;
this.includePrefixes = includePrefixes;

final String prefix;
if (caseSensitive) {

0 comments on commit 7f3da56

Please sign in to comment.