Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.api.gax.rpc.UnaryCallSettings;
import com.google.auth.Credentials;
import com.google.cloud.bigtable.data.v2.internal.channels.MillisTimestampInterceptor;
import com.google.cloud.bigtable.data.v2.models.Query;
import com.google.cloud.bigtable.data.v2.models.Row;
import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings;
Expand All @@ -33,7 +34,6 @@
import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider;
import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import io.grpc.ManagedChannelBuilder;
import java.io.IOException;
import java.util.List;
import java.util.logging.Logger;
Expand Down Expand Up @@ -138,7 +138,13 @@ public static Builder newBuilderForEmulator(String hostname, int port) {
InstantiatingGrpcChannelProvider.newBuilder()
.setMaxInboundMessageSize(256 * 1024 * 1024)
.setChannelPoolSettings(ChannelPoolSettings.staticallySized(1))
.setChannelConfigurator(ManagedChannelBuilder::usePlaintext)
// The emulator only supports millisecond timestamp granularity, so round
// client generated timestamps down the way the service would.
.setChannelConfigurator(
channelBuilder ->
channelBuilder
.usePlaintext()
.intercept(new MillisTimestampInterceptor()))
.setKeepAliveTimeDuration(
java.time.Duration.ofSeconds(61)) // sends ping in this interval
.setKeepAliveTimeoutDuration(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.bigtable.data.v2.internal.channels;

import com.google.bigtable.v2.CheckAndMutateRowRequest;
import com.google.bigtable.v2.MutateRowRequest;
import com.google.bigtable.v2.MutateRowsRequest;
import com.google.bigtable.v2.Mutation;
import com.google.bigtable.v2.Mutation.SetCell;
import com.google.bigtable.v2.Mutation.TimestampOrigin;
import io.grpc.CallOptions;
import io.grpc.Channel;
import io.grpc.ClientCall;
import io.grpc.ClientInterceptor;
import io.grpc.ForwardingClientCall.SimpleForwardingClientCall;
import io.grpc.MethodDescriptor;
import java.util.ArrayList;
import java.util.List;

/**
* Rounds client generated cell timestamps down to millisecond granularity.
*
* <p>{@link com.google.cloud.bigtable.data.v2.models.Mutation#setCell(String, String, String)} and
* its siblings that don't take an explicit timestamp stamp the cell with the current time in
* microseconds and tag it as {@link TimestampOrigin#CLIENT_AUTO_GENERATED}. The Bigtable service
* truncates such timestamps to the granularity of the target table, so writing to a table with the
* default {@code MILLIS} granularity keeps working. The Bigtable emulator predates {@code
* timestamp_origin} and instead rejects any timestamp that is not a multiple of 1000, which breaks
* every auto timestamped write.
*
* <p>This interceptor performs the truncation that the emulator is missing, so that it behaves like
* a production table with millisecond granularity. Timestamps that the caller specified explicitly
* are left untouched: those are rejected by a millisecond granularity table in production too.
*/
public class MillisTimestampInterceptor implements ClientInterceptor {
private static final long MICROS_PER_MILLI = 1_000L;

@Override
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
return new SimpleForwardingClientCall<ReqT, RespT>(next.newCall(method, callOptions)) {
@SuppressWarnings("unchecked")
@Override
public void sendMessage(ReqT message) {
super.sendMessage((ReqT) truncateRequest(message));
}
};
}

private static Object truncateRequest(Object message) {
if (message instanceof MutateRowRequest) {
MutateRowRequest request = (MutateRowRequest) message;
return request.toBuilder()
.clearMutations()
.addAllMutations(truncateAll(request.getMutationsList()))
.build();
} else if (message instanceof MutateRowsRequest) {
MutateRowsRequest request = (MutateRowsRequest) message;
MutateRowsRequest.Builder builder = request.toBuilder().clearEntries();
for (MutateRowsRequest.Entry entry : request.getEntriesList()) {
builder.addEntries(
entry.toBuilder()
.clearMutations()
.addAllMutations(truncateAll(entry.getMutationsList())));
}
return builder.build();
} else if (message instanceof CheckAndMutateRowRequest) {
CheckAndMutateRowRequest request = (CheckAndMutateRowRequest) message;
return request.toBuilder()
.clearTrueMutations()
.addAllTrueMutations(truncateAll(request.getTrueMutationsList()))
.clearFalseMutations()
.addAllFalseMutations(truncateAll(request.getFalseMutationsList()))
.build();
}

return message;
}

private static List<Mutation> truncateAll(List<Mutation> mutations) {
List<Mutation> truncated = new ArrayList<>(mutations.size());
for (Mutation mutation : mutations) {
truncated.add(truncate(mutation));
}
return truncated;
}

/** Returns the mutation with its timestamp truncated, or as is if it didn't need truncating. */
private static Mutation truncate(Mutation mutation) {
if (mutation.getTimestampOrigin() != TimestampOrigin.CLIENT_AUTO_GENERATED
|| !mutation.hasSetCell()) {
return mutation;
}

SetCell setCell = mutation.getSetCell();
long micros = setCell.getTimestampMicros();
long millisAligned = micros - Math.floorMod(micros, MICROS_PER_MILLI);
if (millisAligned == micros) {
return mutation;
}

return mutation.toBuilder()
.setSetCell(setCell.toBuilder().setTimestampMicros(millisAligned))
.build();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.bigtable.data.v2.internal.channels;

import static com.google.common.truth.Truth.assertThat;

import com.google.bigtable.v2.BigtableGrpc;
import com.google.bigtable.v2.CheckAndMutateRowRequest;
import com.google.bigtable.v2.MutateRowRequest;
import com.google.bigtable.v2.MutateRowsRequest;
import com.google.bigtable.v2.Mutation;
import com.google.bigtable.v2.Mutation.DeleteFromRow;
import com.google.bigtable.v2.Mutation.SetCell;
import com.google.bigtable.v2.Mutation.TimestampOrigin;
import com.google.protobuf.ByteString;
import io.grpc.CallOptions;
import io.grpc.Channel;
import io.grpc.ClientCall;
import io.grpc.MethodDescriptor;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

@RunWith(JUnit4.class)
public class MillisTimestampInterceptorTest {
private static final long UNALIGNED_MICROS = 1_234_567_890_123_456L;
private static final long ALIGNED_MICROS = 1_234_567_890_123_000L;

private CapturingChannel channel;
private MillisTimestampInterceptor interceptor;

@Before
public void setUp() {
channel = new CapturingChannel();
interceptor = new MillisTimestampInterceptor();
}

@Test
public void mutateRowTruncatesAutoGeneratedTimestamps() {
MutateRowRequest request =
MutateRowRequest.newBuilder()
.addMutations(setCell(UNALIGNED_MICROS, TimestampOrigin.CLIENT_AUTO_GENERATED))
.build();

MutateRowRequest sent = send(BigtableGrpc.getMutateRowMethod(), request);

assertThat(sent.getMutations(0).getSetCell().getTimestampMicros()).isEqualTo(ALIGNED_MICROS);
}

@Test
public void mutateRowLeavesUserSpecifiedTimestampsAlone() {
MutateRowRequest request =
MutateRowRequest.newBuilder()
.addMutations(setCell(UNALIGNED_MICROS, TimestampOrigin.USER_SPECIFIED))
.build();

assertThat(send(BigtableGrpc.getMutateRowMethod(), request)).isEqualTo(request);
}

@Test
public void mutateRowLeavesNonSetCellMutationsAlone() {
MutateRowRequest request =
MutateRowRequest.newBuilder()
.addMutations(
Mutation.newBuilder()
.setDeleteFromRow(DeleteFromRow.getDefaultInstance())
.setTimestampOrigin(TimestampOrigin.CLIENT_AUTO_GENERATED))
.build();

assertThat(send(BigtableGrpc.getMutateRowMethod(), request)).isEqualTo(request);
}

@Test
public void mutateRowsTruncatesEveryEntry() {
MutateRowsRequest request =
MutateRowsRequest.newBuilder()
.addEntries(
MutateRowsRequest.Entry.newBuilder()
.setRowKey(ByteString.copyFromUtf8("row1"))
.addMutations(setCell(ALIGNED_MICROS, TimestampOrigin.CLIENT_AUTO_GENERATED)))
.addEntries(
MutateRowsRequest.Entry.newBuilder()
.setRowKey(ByteString.copyFromUtf8("row2"))
.addMutations(setCell(UNALIGNED_MICROS, TimestampOrigin.CLIENT_AUTO_GENERATED))
.addMutations(setCell(UNALIGNED_MICROS, TimestampOrigin.USER_SPECIFIED)))
.build();

MutateRowsRequest sent = send(BigtableGrpc.getMutateRowsMethod(), request);

assertThat(sent.getEntries(0)).isEqualTo(request.getEntries(0));
assertThat(sent.getEntries(1).getRowKey()).isEqualTo(ByteString.copyFromUtf8("row2"));
assertThat(sent.getEntries(1).getMutations(0).getSetCell().getTimestampMicros())
.isEqualTo(ALIGNED_MICROS);
assertThat(sent.getEntries(1).getMutations(1).getSetCell().getTimestampMicros())
.isEqualTo(UNALIGNED_MICROS);
}

@Test
public void checkAndMutateRowTruncatesBothBranches() {
CheckAndMutateRowRequest request =
CheckAndMutateRowRequest.newBuilder()
.addTrueMutations(setCell(UNALIGNED_MICROS, TimestampOrigin.CLIENT_AUTO_GENERATED))
.addFalseMutations(setCell(UNALIGNED_MICROS, TimestampOrigin.CLIENT_AUTO_GENERATED))
.build();

CheckAndMutateRowRequest sent = send(BigtableGrpc.getCheckAndMutateRowMethod(), request);

assertThat(sent.getTrueMutations(0).getSetCell().getTimestampMicros())
.isEqualTo(ALIGNED_MICROS);
assertThat(sent.getFalseMutations(0).getSetCell().getTimestampMicros())
.isEqualTo(ALIGNED_MICROS);
}

@Test
public void otherRequestsPassThrough() {
com.google.bigtable.v2.ReadRowsRequest request =
com.google.bigtable.v2.ReadRowsRequest.newBuilder().setTableName("my-table").build();

assertThat(send(BigtableGrpc.getReadRowsMethod(), request)).isEqualTo(request);
}

private static Mutation setCell(long timestampMicros, TimestampOrigin timestampOrigin) {
return Mutation.newBuilder()
.setSetCell(
SetCell.newBuilder()
.setFamilyName("cf")
.setColumnQualifier(ByteString.copyFromUtf8("q"))
.setTimestampMicros(timestampMicros)
.setValue(ByteString.copyFromUtf8("v")))
.setTimestampOrigin(timestampOrigin)
.build();
}

@SuppressWarnings("unchecked")
private <ReqT, RespT> ReqT send(MethodDescriptor<ReqT, RespT> method, ReqT request) {
interceptor.interceptCall(method, CallOptions.DEFAULT, channel).sendMessage(request);
return (ReqT) channel.lastMessage;
}

/** A channel whose calls do nothing except record the last message that was sent. */
private static class CapturingChannel extends Channel {
private Object lastMessage;

@Override
public <ReqT, RespT> ClientCall<ReqT, RespT> newCall(
MethodDescriptor<ReqT, RespT> method, CallOptions callOptions) {
return new ClientCall<ReqT, RespT>() {
@Override
public void start(Listener<RespT> responseListener, io.grpc.Metadata headers) {}

@Override
public void request(int numMessages) {}

@Override
public void cancel(String message, Throwable cause) {}

@Override
public void halfClose() {}

@Override
public void sendMessage(ReqT message) {
lastMessage = message;
}
};
}

@Override
public String authority() {
return "fake-authority";
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,6 @@ public class BulkMutateIT {

@Test(timeout = 60 * 1000)
public void test() throws IOException, InterruptedException {
assume()
.withMessage("Emulator does not support microsecond timestamp granularity")
.that(testEnvRule.env())
.isNotInstanceOf(EmulatorEnv.class);

BigtableDataSettings settings = testEnvRule.env().getDataClientSettings();
String rowPrefix = UUID.randomUUID().toString();
// Set target latency really low so it'll trigger adjusting thresholds
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,6 @@ public class CheckAndMutateIT {

@Test
public void test() throws Exception {
assume()
.withMessage("Emulator does not support microsecond timestamp granularity")
.that(testEnvRule.env())
.isNotInstanceOf(EmulatorEnv.class);

TableId tableId = testEnvRule.env().getTableId();
String familyId = testEnvRule.env().getFamilyId();
String rowKey = UUID.randomUUID().toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ public class MutateRowIT {

@Test
public void test() throws Exception {
assume()
.withMessage("Emulator does not support microsecond timestamp granularity")
.that(testEnvRule.env())
.isNotInstanceOf(EmulatorEnv.class);

String rowKey = UUID.randomUUID().toString();
String familyId = testEnvRule.env().getFamilyId();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,6 @@ public void setUp() {

@Test
public void isRowExists() throws Exception {
assume()
.withMessage("Emulator does not support microsecond timestamp granularity")
.that(testEnvRule.env())
.isNotInstanceOf(EmulatorEnv.class);

String rowKey = prefix + "-test-row-key";
TableId tableId = testEnvRule.env().getTableId();
testEnvRule
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,6 @@ public class SampleRowsIT {

@Test
public void test() throws InterruptedException, ExecutionException, TimeoutException {
assume()
.withMessage("Emulator does not support microsecond timestamp granularity")
.that(testEnvRule.env())
.isNotInstanceOf(EmulatorEnv.class);

BigtableDataClient client = testEnvRule.env().getDataClient();
String rowPrefix = UUID.randomUUID().toString();

Expand Down
Loading