Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import com.google.cloud.storage.Storage.BlobGetOption;
import com.google.cloud.storage.it.runner.StorageITRunner;
import com.google.cloud.storage.it.runner.annotations.Backend;
import com.google.cloud.storage.it.runner.annotations.BucketFixture;
import com.google.cloud.storage.it.runner.annotations.BucketType;
import com.google.cloud.storage.it.runner.annotations.Inject;
import com.google.cloud.storage.it.runner.annotations.SingleBackend;
import com.google.cloud.storage.it.runner.annotations.StorageFixture;
Expand All @@ -37,12 +39,11 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;

@RunWith(StorageITRunner.class)
@SingleBackend(Backend.TEST_BENCH)
@SingleBackend(Backend.PROD)
public final class ITAppendableUploadTest {

@Inject public Generator generator;
Expand All @@ -51,7 +52,9 @@ public final class ITAppendableUploadTest {
@StorageFixture(TransportCompatibility.Transport.GRPC)
public Storage storage;

@Inject public BucketInfo bucket;
@Inject
@BucketFixture(BucketType.RAPID)
public BucketInfo bucket;

@Test
public void testAppendableBlobUpload()
Expand Down Expand Up @@ -92,7 +95,6 @@ public void appendableBlobUploadWithoutFinalizing() throws IOException {
}

@Test
@Ignore("Pending work in testbench, manually verified internally on 2025-03-03")
public void appendableBlobUploadTakeover() throws Exception {
BlobAppendableUploadConfig uploadConfig =
BlobAppendableUploadConfig.of().withFlushPolicy(FlushPolicy.maxFlushSize(5));
Expand Down Expand Up @@ -137,7 +139,6 @@ public void testUploadFileUsingAppendable() throws Exception {
}

@Test
@Ignore("Pending work in testbench, manually verified internally on 2025-03-25")
public void finalizeAfterCloseWorks() throws Exception {
BlobAppendableUploadConfig uploadConfig =
BlobAppendableUploadConfig.of().withFlushPolicy(FlushPolicy.maxFlushSize(1024));
Expand All @@ -153,7 +154,6 @@ public void finalizeAfterCloseWorks() throws Exception {
}

@Test
@Ignore("Pending work in testbench, manually verified internally on 2025-03-25")
public void takeoverJustToFinalizeWorks() throws Exception {
BlobAppendableUploadConfig uploadConfig =
BlobAppendableUploadConfig.of().withFlushPolicy(FlushPolicy.maxFlushSize(5));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,18 @@
import com.google.api.core.ApiFutures;
import com.google.api.gax.rpc.OutOfRangeException;
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
import com.google.cloud.storage.Storage.BlobTargetOption;
import com.google.cloud.storage.Storage.BlobWriteOption;
import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.ZeroCopySupport.DisposableByteString;
import com.google.cloud.storage.it.ChecksummedTestContent;
import com.google.cloud.storage.it.runner.StorageITRunner;
import com.google.cloud.storage.it.runner.annotations.Backend;
import com.google.cloud.storage.it.runner.annotations.BucketFixture;
import com.google.cloud.storage.it.runner.annotations.BucketType;
import com.google.cloud.storage.it.runner.annotations.Inject;
import com.google.cloud.storage.it.runner.annotations.SingleBackend;
import com.google.cloud.storage.it.runner.annotations.StorageFixture;
import com.google.cloud.storage.it.runner.registry.Generator;
import com.google.cloud.storage.it.runner.registry.ObjectsFixture;
import com.google.cloud.storage.it.runner.registry.ObjectsFixture.ObjectAndContent;
import com.google.common.base.Stopwatch;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
Expand All @@ -63,7 +62,7 @@
import org.junit.runner.RunWith;

@RunWith(StorageITRunner.class)
@SingleBackend(Backend.TEST_BENCH)
@SingleBackend(Backend.PROD)
public final class ITObjectReadSessionTest {

private static final int _512KiB = 512 * 1024;
Expand All @@ -72,18 +71,20 @@ public final class ITObjectReadSessionTest {
@StorageFixture(Transport.GRPC)
public Storage storage;

@Inject public BucketInfo bucket;
@Inject
@BucketFixture(BucketType.RAPID)
public BucketInfo bucket;

@Inject public Generator generator;

@Inject public ObjectsFixture objectsFixture;

@Test
public void bytes()
throws ExecutionException, InterruptedException, TimeoutException, IOException {
ObjectAndContent obj512KiB = objectsFixture.getObj512KiB();
byte[] expected = obj512KiB.getContent().getBytes(_512KiB - 13);
BlobId blobId = obj512KiB.getInfo().getBlobId();
ChecksummedTestContent testContent =
ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(512 * 1024));
BlobInfo obj512KiB = create(testContent);
byte[] expected = testContent.getBytes(_512KiB - 13);
BlobId blobId = obj512KiB.getBlobId();

try (BlobReadSession blobReadSession =
storage.blobReadSession(blobId).get(30, TimeUnit.SECONDS)) {
Expand All @@ -105,8 +106,10 @@ public void bytes()

@Test
public void attemptingToGetFutureOutSizeSessionFails() throws Throwable {
ObjectAndContent obj512KiB = objectsFixture.getObj512KiB();
BlobId blobId = obj512KiB.getInfo().getBlobId();
ChecksummedTestContent testContent =
ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(512 * 1024));
BlobInfo obj512KiB = create(testContent);
BlobId blobId = obj512KiB.getBlobId();

ApiFuture<byte[]> future;
try (BlobReadSession session = storage.blobReadSession(blobId).get(30, TimeUnit.SECONDS)) {
Expand Down Expand Up @@ -322,15 +325,11 @@ public void seekable() throws Exception {
@Test
public void outOfRange()
throws ExecutionException, InterruptedException, TimeoutException, IOException {
BlobId blobId;
BlobWriteSession session =
storage.blobWriteSession(
BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(),
BlobWriteOption.doesNotExist());
try (WritableByteChannel upload = session.open()) {
upload.write(DataGenerator.base64Characters().genByteBuffer(4));
}
blobId = session.getResult().get(5, TimeUnit.SECONDS).getBlobId();
ChecksummedTestContent testContent =
ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(4));
BlobInfo obj512KiB = create(testContent);
BlobId blobId = obj512KiB.getBlobId();

try (BlobReadSession blobReadSession =
storage.blobReadSession(blobId).get(30, TimeUnit.SECONDS)) {

Expand All @@ -350,10 +349,14 @@ public void outOfRange()
}
}

private BlobInfo create(ChecksummedTestContent content) {
return storage.create(
BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(),
content.getBytes(),
BlobTargetOption.doesNotExist());
private BlobInfo create(ChecksummedTestContent content)
throws IOException, ExecutionException, InterruptedException, TimeoutException {
BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build();
try (BlobAppendableUpload upload =
storage.blobAppendableUpload(
info, BlobAppendableUploadConfig.of(), BlobWriteOption.doesNotExist())) {
upload.write(ByteBuffer.wrap(content.getBytes()));
return upload.finalizeUpload().get(30, TimeUnit.SECONDS);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,7 @@ public enum BucketType {
/** A bucket created with Hierarchical Namespace enabled */
HNS,
/** A bucket created using all GCS default except that object versioning is enabled */
VERSIONED
VERSIONED,
/** A Rapid bucket */
RAPID
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@
import com.google.api.gax.core.NoCredentialsProvider;
import com.google.cloud.NoCredentials;
import com.google.cloud.storage.BucketInfo;
import com.google.cloud.storage.BucketInfo.CustomPlacementConfig;
import com.google.cloud.storage.BucketInfo.HierarchicalNamespace;
import com.google.cloud.storage.BucketInfo.IamConfiguration;
import com.google.cloud.storage.GrpcStorageOptions;
import com.google.cloud.storage.HttpStorageOptions;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageClass;
import com.google.cloud.storage.StorageOptions;
import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.it.GrpcPlainRequestLoggingInterceptor;
Expand Down Expand Up @@ -242,6 +244,31 @@ static BackendResources of(
storageJson.get().getStorage(),
ctrl.get().getCtrl());
});
TestRunScopedInstance<BucketInfoShim> bucketRapid =
TestRunScopedInstance.of(
"fixture/BUCKET/[" + backend.name() + "]/RAPID",
() -> {
String bucketName =
String.format(Locale.US, "java-storage-grpc-rapid-%s", UUID.randomUUID());
protectedBucketNames.add(bucketName);
return new BucketInfoShim(
BucketInfo.newBuilder(bucketName)
.setLocation(zone.get().get().getRegion())
.setCustomPlacementConfig(
CustomPlacementConfig.newBuilder()
.setDataLocations(ImmutableList.of(zone.get().get().getZone()))
.build())
.setStorageClass(StorageClass.valueOf("RAPID"))
.setHierarchicalNamespace(
HierarchicalNamespace.newBuilder().setEnabled(true).build())
.setIamConfiguration(
IamConfiguration.newBuilder()
.setIsUniformBucketLevelAccessEnabled(true)
.build())
.build(),
storageJson.get().getStorage(),
ctrl.get().getCtrl());
});
TestRunScopedInstance<ObjectsFixture> objectsFixture =
TestRunScopedInstance.of(
"fixture/OBJECTS/[" + backend.name() + "]",
Expand Down Expand Up @@ -286,6 +313,11 @@ static BackendResources of(
BucketInfo.class,
bucketVersioned,
backendIs(backend).and(bucketTypeIs(BucketType.VERSIONED))),
RegistryEntry.of(
63,
BucketInfo.class,
bucketRapid,
backendIs(backend).and(bucketTypeIs(BucketType.RAPID))),
RegistryEntry.of(
70, BucketInfo.class, bucket, backendIs(backend).and(isDefaultBucket())),
RegistryEntry.of(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,11 @@

package com.google.cloud.storage.it.runner.registry;

import static org.junit.Assume.assumeTrue;

import com.google.cloud.storage.BucketInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.it.BucketCleaner;
import com.google.storage.control.v2.StorageControlClient;

Expand Down Expand Up @@ -47,7 +50,16 @@ public Object get() {

@Override
public void start() {
createdBucket = s.create(bucketInfo).asBucketInfo();
try {
createdBucket = s.create(bucketInfo).asBucketInfo();
} catch (StorageException se) {
String msg = se.getMessage();
if (se.getCode() == 400 && msg.contains("not a valid zone in location")) {
assumeTrue(
"Skipping test due to bucket setup unavailable in current zone. (" + msg + ")", false);
}
throw se;
}
}

@Override
Expand Down