Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add ability to create a PCU Prefix at the object level #2345

Merged
merged 9 commits into from
Jan 19, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -462,10 +462,7 @@ String fmtName(String ultimateObjectName, PartRange partRange) {

// encode it to base 64, yielding 22 characters
String randomKey = B64.encodeToString(bytes);
HashCode hashCode =
OBJECT_NAME_HASH_FUNCTION.hashString(ultimateObjectName, StandardCharsets.UTF_8);
String nameDigest = B64.encodeToString(hashCode.asBytes());
return fmtFields(randomKey, nameDigest, partRange.encode());
return fmtFields(randomKey, ultimateObjectName, partRange.encode());
}

abstract String fmtFields(String randomKey, String nameDigest, String partRange);
Expand Down Expand Up @@ -523,6 +520,41 @@ public static PartNamingStrategy prefix(String prefixPattern) {
return new WithPrefix(rand, prefixPattern);
}

@BetaApi
public static PartNamingStrategy objectLevelPrefix() {
return objectLevelPrefix("");
}

/**
* Strategy in which an explicit stable prefix with object name included is present on each part
* and intermediary compose object.
*
* <p>General format is
*
* <pre><code>
* {prefixPattern}/{objectName}/{randomKeyDigest};{objectInfoDigest};{partIndex}.part
* </code></pre>
*
* <p>{@code {objectInfoDigest}} will be fixed for an individual {@link BlobWriteSession}.
*
* <p><b><i>NOTE:</i></b>The way in which both {@code randomKeyDigest} and {@code
* objectInfoDigest} are generated is undefined and subject to change at any time.
*
* <p>Care must be taken when choosing to specify a stable prefix as this can create hotspots in
* the keyspace for object names. See <a
* href="https://cloud.google.com/storage/docs/request-rate#naming-convention">Object Naming
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
* Convention Guidelines</a> for more details.
*
* @see #withPartNamingStrategy(PartNamingStrategy)
* @since 2.30.2 This new api is in preview and is subject to breaking changes.
*/
@BetaApi
public static PartNamingStrategy objectLevelPrefix(String prefixPattern) {
checkNotNull(prefixPattern, "prefixPattern must be non null");
SecureRandom rand = new SecureRandom();
return new WithObjectLevelPrefix(rand, prefixPattern);
}

static final class WithPrefix extends PartNamingStrategy {
private static final long serialVersionUID = 5709330763161570411L;

Expand All @@ -534,8 +566,40 @@ private WithPrefix(SecureRandom rand, String prefix) {
}

@Override
protected String fmtFields(String randomKey, String nameDigest, String partRange) {
protected String fmtFields(String randomKey, String ultimateObjectName, String partRange) {
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
HashCode hashCode =
OBJECT_NAME_HASH_FUNCTION.hashString(ultimateObjectName, StandardCharsets.UTF_8);
String nameDigest = B64.encodeToString(hashCode.asBytes());
return prefix
+ "/"
+ randomKey
+ FIELD_SEPARATOR
+ nameDigest
+ FIELD_SEPARATOR
+ partRange
+ ".part";
}
}

static final class WithObjectLevelPrefix extends PartNamingStrategy {

private static final long serialVersionUID = 5157942020618764450L;
private final String prefix;

private WithObjectLevelPrefix(SecureRandom rand, String prefix) {
super(rand);
// If no prefix is specified we will create the part files under the same directory as the
// ultimate object.
this.prefix = prefix.isEmpty() ? prefix : prefix + "/";
}

@Override
protected String fmtFields(String randomKey, String ultimateObjectName, String partRange) {
HashCode hashCode =
OBJECT_NAME_HASH_FUNCTION.hashString(ultimateObjectName, StandardCharsets.UTF_8);
String nameDigest = B64.encodeToString(hashCode.asBytes());
return prefix
+ ultimateObjectName
+ "/"
+ randomKey
+ FIELD_SEPARATOR
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,38 @@ public void partNameStrategy_prefix_stillWorksWithFmtPattern() throws Exception
() -> assertThat(fmt).startsWith("[%s]/"));
}

@Test
public void partNameStrategy_prefixObjectLevel() throws Exception {
PartNamingStrategy strategy = PartNamingStrategy.objectLevelPrefix("asdf");

String fmt = strategy.fmtName("a/b/obj", PartRange.of(1, 96));
assertAll(
// random digest with prefix to spread over keyspace
// digest is 22, prefix is 4, slash is 1 , objectName is 7, additional slash is 1
() -> assertField(fmt, 0).hasLength(22 + 5 + 8),
// name digest
() -> assertField(fmt, 1).hasLength(22),
() -> assertField(fmt, 2).isEqualTo("0001-0096.part"),
() -> assertThat(fmt).startsWith("asdf/a/b/obj/"));
}

@Test
public void partNameStrategy_noPrefixObjectLevel() throws Exception {
// Creating an object level prefix without specifying an additional prefix will append the
// object name to the beginning of the part name.
PartNamingStrategy strategy = PartNamingStrategy.objectLevelPrefix();

String fmt = strategy.fmtName("a/b/obj", PartRange.of(1, 96));
assertAll(
// random digest with prefix to spread over keyspace
// digest is 22, objectName is 7, slash is 1
() -> assertField(fmt, 0).hasLength(22 + 8),
// name digest
() -> assertField(fmt, 1).hasLength(22),
() -> assertField(fmt, 2).isEqualTo("0001-0096.part"),
() -> assertThat(fmt).startsWith("a/b/obj/"));
}

private static StringSubject assertField(String fmt, int idx) {
String[] split = fmt.split(";");
String s = split[idx];
Expand Down