From 4bd0714b39bda7e3a8a3ca1893cf8955872f05f7 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 30 Mar 2017 15:13:11 -0700 Subject: [PATCH 01/21] CPKAFKA-465 : Implement READ_COMMITTED mode in the KafkaConsumer (#145) --- .../clients/consumer/ConsumerConfig.java | 17 ++ .../kafka/clients/consumer/KafkaConsumer.java | 13 +- .../clients/consumer/internals/Fetcher.java | 91 +++++- .../kafka/common/record/MemoryRecords.java | 10 +- .../kafka/common/requests/FetchRequest.java | 21 +- .../common/requests/ListOffsetRequest.java | 1 + .../clients/consumer/KafkaConsumerTest.java | 7 +- .../consumer/internals/FetcherTest.java | 277 +++++++++++++++++- .../unit/kafka/server/FetchRequestTest.scala | 5 +- 9 files changed, 415 insertions(+), 27 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index abb8255774fbc..930c115406fce 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.config.ConfigDef.Importance; import org.apache.kafka.common.config.ConfigDef.Type; import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.requests.IsolationLevel; import org.apache.kafka.common.serialization.Deserializer; import java.util.Collections; @@ -234,6 +235,17 @@ public class ConsumerConfig extends AbstractConfig { */ static final String LEAVE_GROUP_ON_CLOSE_CONFIG = "internal.leave.group.on.close"; + /** isolation.level */ + public static final String ISOLATION_LEVEL_CONFIG = "isolation.level"; + public static final String ISOLATION_LEVEL_DOC = "

Controls how to read messages written transactionally. If set to READ_COMMITTED, consumer.poll() will only return" + + " transactional messages which have been committed. If set to READ_UNCOMMITTED' (the default), consumer.poll() will return all messages, even transactional messages" + + " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.

Messages will be always returned in offset order. Hence, in " + + " READ_COMMITTED mode, consumer.poll() will only return messages upto the last resolved (committed or aborted) transaction. In particular any messages appearing after" + + " messages belonging onging transactions will be withheld until the said transaction has been completed and its messages are delivered to the application. As a result, READ_COMMITTED" + + " consumers will not be able to read upto the log end offset when there are inflight transactions.

"; + + public static final String DEFAULT_ISOLATION_LEVEL = IsolationLevel.READ_UNCOMMITTED.toString(); + static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, @@ -405,6 +417,11 @@ public class ConsumerConfig extends AbstractConfig { Type.BOOLEAN, true, Importance.LOW) + .define(ISOLATION_LEVEL_CONFIG, + Type.STRING, + DEFAULT_ISOLATION_LEVEL, + Importance.LOW, + ISOLATION_LEVEL_DOC) // security support .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, Type.STRING, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 9c703b511984a..fba0ae1c143d0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -43,6 +43,7 @@ import org.apache.kafka.common.metrics.MetricsReporter; import org.apache.kafka.common.network.ChannelBuilder; import org.apache.kafka.common.network.Selector; +import org.apache.kafka.common.requests.IsolationLevel; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.utils.AppInfoParser; @@ -532,6 +533,7 @@ public class KafkaConsumer implements Consumer { private final Metadata metadata; private final long retryBackoffMs; private final long requestTimeoutMs; + private final IsolationLevel isolationLevel; private volatile boolean closed = false; // currentThread holds the threadId of the current thread accessing KafkaConsumer @@ -660,6 +662,10 @@ private KafkaConsumer(ConsumerConfig config, this.metadata.update(Cluster.bootstrap(addresses), Collections.emptySet(), 0); String metricGrpPrefix = "consumer"; ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); + + this.isolationLevel = IsolationLevel.valueOf( + config.getString(ConsumerConfig.ISOLATION_LEVEL_CONFIG).toUpperCase(Locale.ENGLISH)); + NetworkClient netClient = new NetworkClient( new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), this.metadata, @@ -710,7 +716,8 @@ private KafkaConsumer(ConsumerConfig config, metrics, metricGrpPrefix, this.time, - this.retryBackoffMs); + this.retryBackoffMs, + IsolationLevel.READ_UNCOMMITTED); config.logUnused(); AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); @@ -738,7 +745,8 @@ private KafkaConsumer(ConsumerConfig config, SubscriptionState subscriptions, Metadata metadata, long retryBackoffMs, - long requestTimeoutMs) { + long requestTimeoutMs, + String isolationLevel) { this.clientId = clientId; this.coordinator = coordinator; this.keyDeserializer = keyDeserializer; @@ -752,6 +760,7 @@ private KafkaConsumer(ConsumerConfig config, this.metadata = metadata; this.retryBackoffMs = retryBackoffMs; this.requestTimeoutMs = requestTimeoutMs; + this.isolationLevel = IsolationLevel.valueOf(isolationLevel.toUpperCase(Locale.ENGLISH)); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index f421dfb6b8147..df96c3d792f4c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -29,6 +29,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.CorruptRecordException; import org.apache.kafka.common.errors.InvalidMetadataException; import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.errors.RecordTooLargeException; @@ -44,12 +45,14 @@ import org.apache.kafka.common.metrics.stats.Rate; import org.apache.kafka.common.metrics.stats.Value; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.ControlRecordType; import org.apache.kafka.common.record.InvalidRecordException; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; +import org.apache.kafka.common.requests.IsolationLevel; import org.apache.kafka.common.requests.ListOffsetRequest; import org.apache.kafka.common.requests.ListOffsetResponse; import org.apache.kafka.common.requests.MetadataRequest; @@ -66,16 +69,21 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.PriorityQueue; +import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; +import static java.util.Collections.emptyList; + /** * This class manage the fetching process with the brokers. */ @@ -98,6 +106,8 @@ public class Fetcher implements SubscriptionState.Listener, Closeable { private final ConcurrentLinkedQueue completedFetches; private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; + private final IsolationLevel isolationLevel; + private PartitionRecords nextInLineRecords = null; private ExceptionMetadata nextInLineExceptionMetadata = null; @@ -115,7 +125,8 @@ public Fetcher(ConsumerNetworkClient client, Metrics metrics, String metricGrpPrefix, Time time, - long retryBackoffMs) { + long retryBackoffMs, + IsolationLevel isolationLevel) { this.time = time; this.client = client; this.metadata = metadata; @@ -131,6 +142,7 @@ public Fetcher(ConsumerNetworkClient client, this.completedFetches = new ConcurrentLinkedQueue<>(); this.sensors = new FetchManagerMetrics(metrics, metricGrpPrefix); this.retryBackoffMs = retryBackoffMs; + this.isolationLevel = isolationLevel; subscriptions.addListener(this); } @@ -370,7 +382,7 @@ private long offsetResetStrategyTimestamp(final TopicPartition partition) { if (strategy == OffsetResetStrategy.EARLIEST) timestamp = ListOffsetRequest.EARLIEST_TIMESTAMP; else if (strategy == OffsetResetStrategy.LATEST) - timestamp = ListOffsetRequest.LATEST_TIMESTAMP; + timestamp = isolationLevel == IsolationLevel.READ_COMMITTED ? ListOffsetRequest.LSO_TIMESTAMP : ListOffsetRequest.LATEST_TIMESTAMP; else throw new NoOffsetForPartitionException(partition); return timestamp; @@ -457,7 +469,8 @@ public Map beginningOffsets(Collection par } public Map endOffsets(Collection partitions, long timeout) { - return beginningOrEndOffset(partitions, ListOffsetRequest.LATEST_TIMESTAMP, timeout); + long endTimestamp = isolationLevel == IsolationLevel.READ_COMMITTED ? ListOffsetRequest.LSO_TIMESTAMP : ListOffsetRequest.LATEST_TIMESTAMP; + return beginningOrEndOffset(partitions, endTimestamp, timeout); } private Map beginningOrEndOffset(Collection partitions, @@ -570,7 +583,7 @@ private List> fetchRecords(PartitionRecords partitionRecord } partitionRecords.drain(); - return Collections.emptyList(); + return emptyList(); } /** @@ -771,7 +784,7 @@ private Map createFetchRequests() { Map requests = new HashMap<>(); for (Map.Entry> entry : fetchable.entrySet()) { Node node = entry.getKey(); - FetchRequest.Builder fetch = FetchRequest.Builder.forConsumer(this.maxWaitMs, this.minBytes, entry.getValue()). + FetchRequest.Builder fetch = FetchRequest.Builder.forConsumer(this.maxWaitMs, this.minBytes, entry.getValue(), isolationLevel). setMaxBytes(this.maxBytes); requests.put(node, fetch); } @@ -906,6 +919,8 @@ private class PartitionRecords { private final TopicPartition partition; private final CompletedFetch completedFetch; private final Iterator batches; + private final Set abortedPids; + private final Queue abortedTransactions; private int recordsRead; private int bytesRead; @@ -921,6 +936,8 @@ private PartitionRecords(TopicPartition partition, this.completedFetch = completedFetch; this.batches = batches; this.nextFetchOffset = completedFetch.fetchedOffset; + this.abortedPids = new HashSet<>(); + this.abortedTransactions = abortedTransactions(completedFetch.partitionData); } private void drain() { @@ -960,8 +977,10 @@ private void maybeEnsureValid(Record record) { } private void maybeCloseRecordStream() { - if (records != null) + if (records != null) { records.close(); + records = null; + } } private Record nextFetchedRecord() { @@ -974,6 +993,10 @@ private Record nextFetchedRecord() { return null; } currentBatch = batches.next(); + if (isBatchAborted(currentBatch, abortedPids, abortedTransactions)) { + continue; + } + maybeEnsureValid(currentBatch); records = currentBatch.streamingIterator(); } @@ -1008,6 +1031,62 @@ private List> fetchRecords(int maxRecords) { } return records; } + + private boolean isBatchAborted(RecordBatch batch, Set abortedPids, Queue abortedTransactions) { + /* When in READ_COMMITTED mode, we need to do the following for each incoming entry: + * 0. Check whether the pid is in the 'abortedPids' set && the entry does not include an abort marker. + * If so, skip the entry. + * 1. If the pid is in aborted pids and the entry contains an abort marker, remove the pid from + * aborted pids and skip the entry. + * 2. Check lowest offset entry in the abort index. If the PID of the current entry matches the + * pid of the abort index entry, and the incoming offset is no smaller than the abort index offset, + * this means that the entry has been aborted. Add the pid to the aborted pids set, and remove + * the entry from the abort index. + */ + if (isolationLevel == IsolationLevel.READ_COMMITTED) { + FetchResponse.AbortedTransaction nextAbortedTransaction = abortedTransactions.peek(); + if (abortedPids.contains(batch.producerId()) + || (nextAbortedTransaction != null && nextAbortedTransaction.pid == batch.producerId() && nextAbortedTransaction.firstOffset <= batch.baseOffset())) { + if (abortedPids.contains(batch.producerId()) && containsAbortMarker(batch)) { + abortedPids.remove(batch.producerId()); + } else if (nextAbortedTransaction != null && nextAbortedTransaction.pid == batch.producerId() && nextAbortedTransaction.firstOffset <= batch.baseOffset()) { + abortedPids.add(batch.producerId()); + abortedTransactions.remove(nextAbortedTransaction); + } + log.trace("Skipping aborted record with pid {} and base offset {}", batch.producerId(), batch.baseOffset()); + return true; + } + } + return false; + } + + private Queue abortedTransactions(FetchResponse.PartitionData partition) { + PriorityQueue abortedTransactions = null; + if (partition.abortedTransactions != null && 0 < partition.abortedTransactions.size()) { + abortedTransactions = new PriorityQueue<>( + partition.abortedTransactions.size(), + new Comparator() { + @Override + public int compare(FetchResponse.AbortedTransaction o1, FetchResponse.AbortedTransaction o2) { + return (int) o1.firstOffset - (int) o2.firstOffset; + } + } + ); + abortedTransactions.addAll(partition.abortedTransactions); + } else { + abortedTransactions = new PriorityQueue<>(); + } + return abortedTransactions; + } + + private boolean containsAbortMarker(RecordBatch batch) { + Iterator batchIterator = batch.iterator(); + Record firstRecord = batchIterator.hasNext() ? batchIterator.next() : null; + if (firstRecord != null && batchIterator.hasNext()) + throw new CorruptRecordException("A RecordBatch containing a control message contained more than one message."); + return firstRecord != null && firstRecord.isControlRecord() && ControlRecordType.ABORT == ControlRecordType.parse(firstRecord.key()); + } + } private static class ExceptionMetadata { diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java index e7e155f6b3875..5c32bd59a04e0 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java @@ -310,7 +310,7 @@ public static MemoryRecordsBuilder builder(ByteBuffer buffer, long baseOffset, long logAppendTime) { return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime, - RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, RecordBatch.NO_PARTITION_LEADER_EPOCH); + RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, RecordBatch.NO_PARTITION_LEADER_EPOCH); } public static MemoryRecordsBuilder builder(ByteBuffer buffer, @@ -321,7 +321,7 @@ public static MemoryRecordsBuilder builder(ByteBuffer buffer, long logAppendTime, int partitionLeaderEpoch) { return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime, - RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, partitionLeaderEpoch); + RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, partitionLeaderEpoch); } public static MemoryRecordsBuilder builder(ByteBuffer buffer, @@ -334,7 +334,7 @@ public static MemoryRecordsBuilder builder(ByteBuffer buffer, short producerEpoch, int baseSequence) { return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime, - producerId, producerEpoch, baseSequence, RecordBatch.NO_PARTITION_LEADER_EPOCH); + producerId, producerEpoch, baseSequence, false, RecordBatch.NO_PARTITION_LEADER_EPOCH); } public static MemoryRecordsBuilder builder(ByteBuffer buffer, @@ -346,12 +346,14 @@ public static MemoryRecordsBuilder builder(ByteBuffer buffer, long producerId, short producerEpoch, int baseSequence, + boolean isTransactional, int partitionLeaderEpoch) { return new MemoryRecordsBuilder(buffer, magic, compressionType, timestampType, baseOffset, - logAppendTime, producerId, producerEpoch, baseSequence, false, partitionLeaderEpoch, + logAppendTime, producerId, producerEpoch, baseSequence, isTransactional, partitionLeaderEpoch, buffer.remaining()); } + public static MemoryRecords withRecords(CompressionType compressionType, SimpleRecord... records) { return withRecords(RecordBatch.CURRENT_MAGIC_VALUE, compressionType, records); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 8cd281887bd31..78715be3c3917 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -104,24 +104,30 @@ public static class Builder extends AbstractRequest.Builder { private final int minBytes; private final int replicaId; private final LinkedHashMap fetchData; + private final IsolationLevel isolationLevel; private int maxBytes = DEFAULT_RESPONSE_MAX_BYTES; public static Builder forConsumer(int maxWait, int minBytes, LinkedHashMap fetchData) { - return new Builder(null, CONSUMER_REPLICA_ID, maxWait, minBytes, fetchData); + return new Builder(null, CONSUMER_REPLICA_ID, maxWait, minBytes, fetchData, IsolationLevel.READ_UNCOMMITTED); + } + + public static Builder forConsumer(int maxWait, int minBytes, LinkedHashMap fetchData, IsolationLevel isolationLevel) { + return new Builder(null, CONSUMER_REPLICA_ID, maxWait, minBytes, fetchData, isolationLevel); } public static Builder forReplica(short desiredVersion, int replicaId, int maxWait, int minBytes, LinkedHashMap fetchData) { - return new Builder(desiredVersion, replicaId, maxWait, minBytes, fetchData); + return new Builder(desiredVersion, replicaId, maxWait, minBytes, fetchData, IsolationLevel.READ_UNCOMMITTED); } private Builder(Short desiredVersion, int replicaId, int maxWait, int minBytes, - LinkedHashMap fetchData) { + LinkedHashMap fetchData, IsolationLevel isolationLevel) { super(ApiKeys.FETCH, desiredVersion); this.replicaId = replicaId; this.maxWait = maxWait; this.minBytes = minBytes; this.fetchData = fetchData; + this.isolationLevel = isolationLevel; } public LinkedHashMap fetchData() { @@ -139,7 +145,7 @@ public FetchRequest build(short version) { maxBytes = DEFAULT_RESPONSE_MAX_BYTES; } - return new FetchRequest(version, replicaId, maxWait, minBytes, maxBytes, fetchData); + return new FetchRequest(version, replicaId, maxWait, minBytes, maxBytes, fetchData, IsolationLevel.READ_UNCOMMITTED); } @Override @@ -158,13 +164,18 @@ public String toString() { private FetchRequest(short version, int replicaId, int maxWait, int minBytes, int maxBytes, LinkedHashMap fetchData) { + this(version, replicaId, maxWait, minBytes, maxBytes, fetchData, IsolationLevel.READ_UNCOMMITTED); + } + + private FetchRequest(short version, int replicaId, int maxWait, int minBytes, int maxBytes, + LinkedHashMap fetchData, IsolationLevel isolationLevel) { super(version); this.replicaId = replicaId; this.maxWait = maxWait; this.minBytes = minBytes; this.maxBytes = maxBytes; this.fetchData = fetchData; - this.isolationLevel = IsolationLevel.READ_UNCOMMITTED; + this.isolationLevel = isolationLevel; } public FetchRequest(Struct struct, short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java index 33270717ef26e..feb3635b2f52d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java @@ -33,6 +33,7 @@ import java.util.Set; public class ListOffsetRequest extends AbstractRequest { + public static final long LSO_TIMESTAMP = -3L; public static final long EARLIEST_TIMESTAMP = -2L; public static final long LATEST_TIMESTAMP = -1L; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index b9600b44abcb4..1ee1953a04fff 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -49,6 +49,7 @@ import org.apache.kafka.common.requests.FetchResponse.PartitionData; import org.apache.kafka.common.requests.FindCoordinatorResponse; import org.apache.kafka.common.requests.HeartbeatResponse; +import org.apache.kafka.common.requests.IsolationLevel; import org.apache.kafka.common.requests.JoinGroupRequest; import org.apache.kafka.common.requests.JoinGroupResponse; import org.apache.kafka.common.requests.LeaveGroupResponse; @@ -1548,7 +1549,8 @@ private KafkaConsumer newConsumer(Time time, metrics, metricGroupPrefix, time, - retryBackoffMs); + retryBackoffMs, + IsolationLevel.READ_UNCOMMITTED); return new KafkaConsumer<>( clientId, @@ -1563,7 +1565,8 @@ private KafkaConsumer newConsumer(Time time, subscriptions, metadata, retryBackoffMs, - requestTimeoutMs); + requestTimeoutMs, + "READ_UNCOMMITTED"); } private static class FetchInfo { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index b8f493a7df3ef..911abcf5ca78f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -48,6 +48,8 @@ import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.IsolationLevel; +import org.apache.kafka.common.utils.ByteBufferOutputStream; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; @@ -59,7 +61,6 @@ import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.utils.ByteBufferOutputStream; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; @@ -69,13 +70,16 @@ import java.io.DataOutputStream; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static java.util.Collections.singleton; import static org.junit.Assert.assertArrayEquals; @@ -1105,6 +1109,256 @@ public void testGetOffsetsForTimes() { testGetOffsetsForTimesWithError(Errors.BROKER_NOT_AVAILABLE, Errors.NONE, 10L, 100L, 10L, 100L); } + @Test + public void testSkippingAbortedTransactions() { + Fetcher fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds()); + + buffer.flip(); + + List abortedTransactions = new ArrayList<>(); + abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + subscriptions.assignFromUser(singleton(tp1)); + + subscriptions.seek(tp1, 0); + + // normal fetch + assertEquals(1, fetcher.sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(0); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetcher.fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + } + + @Test + public void testReturnCommittedTransactions() { + Fetcher fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + currentOffset += commitTransaction(buffer, 1L, currentOffset, time.milliseconds()); + buffer.flip(); + + List abortedTransactions = new ArrayList<>(); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + subscriptions.assignFromUser(singleton(tp1)); + + subscriptions.seek(tp1, 0); + + // normal fetch + assertEquals(1, fetcher.sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(0); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetcher.fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp1)); + assertEquals(fetchedRecords.get(tp1).size(), 2); + } + + @Test + public void testWithCommittedAndAbortedTransactions() { + Fetcher fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + + List abortedTransactions = new ArrayList<>(); + + int currOffset = 0; + // Appends for producer 1 (evetually committed) + currOffset += appendTransactionalRecords(buffer, 1L, currOffset, + new SimpleRecord(time.milliseconds(), "commit1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "commit1-2".getBytes(), "value".getBytes())); + + // Appends for producer 2 (eventually aborted) + currOffset += appendTransactionalRecords(buffer, 2L, currOffset, + new SimpleRecord(time.milliseconds(), "abort2-1".getBytes(), "value".getBytes())); + + // commit producer 1 + currOffset += commitTransaction(buffer, 1L, currOffset, time.milliseconds()); + // append more for producer 2 (eventually aborted) + currOffset += appendTransactionalRecords(buffer, 2L, currOffset, + new SimpleRecord(time.milliseconds(), "abort2-2".getBytes(), "value".getBytes())); + + // abort producer 2 + currOffset += abortTransaction(buffer, 2L, currOffset, time.milliseconds()); + abortedTransactions.add(new FetchResponse.AbortedTransaction(2, 2)); + + // New transaction for producer 1 (eventually aborted) + currOffset += appendTransactionalRecords(buffer, 1L, currOffset, + new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes())); + + // New transaction for producer 2 (eventually committed) + currOffset += appendTransactionalRecords(buffer, 2L, currOffset, + new SimpleRecord(time.milliseconds(), "commit2-1".getBytes(), "value".getBytes())); + + // Add messages for producer 1 (eventually aborted) + currOffset += appendTransactionalRecords(buffer, 1L, currOffset, + new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); + + // abort producer 1 + currOffset += abortTransaction(buffer, 1L, currOffset, time.milliseconds()); + abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 6)); + + // commit producer 2 + currOffset += commitTransaction(buffer, 2L, currOffset, time.milliseconds()); + + buffer.flip(); + + MemoryRecords records = MemoryRecords.readableRecords(buffer); + subscriptions.assignFromUser(singleton(tp1)); + + subscriptions.seek(tp1, 0); + + // normal fetch + assertEquals(1, fetcher.sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(0); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetcher.fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp1)); + // There are only 3 committed records + List> fetchedConsumerRecords = fetchedRecords.get(tp1); + Set committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2", "commit2-1")); + Set actuallyCommittedKeys = new HashSet<>(); + for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { + actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); + } + assertTrue(actuallyCommittedKeys.equals(committedKeys)); + } + + @Test + public void testMultipleAbortMarkers() { + Fetcher fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds()); + // Duplicate abort -- should be ignored. + currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds()); + // Now commit a transaction. + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "commit1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "commit1-2".getBytes(), "value".getBytes())); + currentOffset += commitTransaction(buffer, 1L, currentOffset, time.milliseconds()); + buffer.flip(); + + List abortedTransactions = new ArrayList<>(); + abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + subscriptions.assignFromUser(singleton(tp1)); + + subscriptions.seek(tp1, 0); + + // normal fetch + assertEquals(1, fetcher.sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(0); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetcher.fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp1)); + assertEquals(fetchedRecords.get(tp1).size(), 2); + List> fetchedConsumerRecords = fetchedRecords.get(tp1); + Set committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2")); + Set actuallyCommittedKeys = new HashSet<>(); + for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { + actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); + } + assertTrue(actuallyCommittedKeys.equals(committedKeys)); + } + + @Test + public void testReturnAbortedTransactionsinUncommittedMode() { + Fetcher fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds()); + + buffer.flip(); + + List abortedTransactions = new ArrayList<>(); + abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + subscriptions.assignFromUser(singleton(tp1)); + + subscriptions.seek(tp1, 0); + + // normal fetch + assertEquals(1, fetcher.sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(0); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetcher.fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp1)); + } + + private int appendTransactionalRecords(ByteBuffer buffer, long pid, int baseOffset, SimpleRecord... records) { + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true); + + for (SimpleRecord record : records) { + builder.append(record); + } + builder.build(); + return records.length; + } + + private int commitTransaction(ByteBuffer buffer, long pid, int baseOffset, long timestamp) { + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true); + builder.appendControlRecord(timestamp, ControlRecordType.COMMIT, null); + builder.build(); + return 1; + } + + private int abortTransaction(ByteBuffer buffer, long pid, int baseOffset, long timestamp) { + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true); + builder.appendControlRecord(timestamp, ControlRecordType.ABORT, null); + builder.build(); + return 1; + } + private void testGetOffsetsForTimesWithError(Errors errorForTp0, Errors errorForTp1, long offsetForTp0, @@ -1167,6 +1421,15 @@ private ListOffsetResponse listOffsetResponse(TopicPartition tp, Errors error, l return new ListOffsetResponse(allPartitionData); } + private FetchResponse fetchResponseWithAbortedTransactions(MemoryRecords records, + List abortedTransactions, + Errors error, + long lastStableOffset, long hw, int throttleTime) { + Map partitions = Collections.singletonMap(tp1, + new FetchResponse.PartitionData(error, hw, lastStableOffset, 0L, abortedTransactions, records)); + return new FetchResponse(new LinkedHashMap<>(partitions), throttleTime); + } + private FetchResponse fetchResponse(MemoryRecords records, Errors error, long hw, int throttleTime) { Map partitions = Collections.singletonMap(tp1, new FetchResponse.PartitionData(error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); @@ -1193,7 +1456,8 @@ private MetadataResponse newMetadataResponse(String topic, Errors error) { private Fetcher createFetcher(SubscriptionState subscriptions, Metrics metrics, int maxPollRecords) { - return createFetcher(subscriptions, metrics, new ByteArrayDeserializer(), new ByteArrayDeserializer(), maxPollRecords); + return createFetcher(subscriptions, metrics, new ByteArrayDeserializer(), new ByteArrayDeserializer(), + maxPollRecords, IsolationLevel.READ_UNCOMMITTED); } private Fetcher createFetcher(SubscriptionState subscriptions, Metrics metrics) { @@ -1204,14 +1468,16 @@ private Fetcher createFetcher(SubscriptionState subscriptions, Metrics metrics, Deserializer keyDeserializer, Deserializer valueDeserializer) { - return createFetcher(subscriptions, metrics, keyDeserializer, valueDeserializer, Integer.MAX_VALUE); + return createFetcher(subscriptions, metrics, keyDeserializer, valueDeserializer, Integer.MAX_VALUE, + IsolationLevel.READ_UNCOMMITTED); } private Fetcher createFetcher(SubscriptionState subscriptions, Metrics metrics, Deserializer keyDeserializer, Deserializer valueDeserializer, - int maxPollRecords) { + int maxPollRecords, + IsolationLevel isolationLevel) { return new Fetcher<>(consumerClient, minBytes, maxBytes, @@ -1226,7 +1492,8 @@ private Fetcher createFetcher(SubscriptionState subscriptions, metrics, "consumer" + groupId, time, - retryBackoffMs); + retryBackoffMs, + isolationLevel); } } diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala index eefb35e9c95b9..99aca99c833e3 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala @@ -26,7 +26,7 @@ import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record.Record -import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} +import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, IsolationLevel} import org.apache.kafka.common.serialization.StringSerializer import org.junit.Assert._ import org.junit.Test @@ -156,8 +156,7 @@ class FetchRequestTest extends BaseRequestTest { val (topicPartition, leaderId) = createTopics(numTopics = 1, numPartitions = 1).head producer.send(new ProducerRecord(topicPartition.topic, topicPartition.partition, "key", new String(new Array[Byte](maxPartitionBytes + 1)))).get - val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, - createPartitionMap(maxPartitionBytes, Seq(topicPartition))).build(2) + val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(maxPartitionBytes, Seq(topicPartition))).build(2) val fetchResponse = sendFetchRequest(leaderId, fetchRequest, version = 2) val partitionData = fetchResponse.responseData.get(topicPartition) assertEquals(Errors.NONE, partitionData.error) From d81b332e1c8f7ac04bb76c88709c37c19c900f9e Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 11 Apr 2017 11:12:26 -0700 Subject: [PATCH 02/21] Complete implementation of the transactional producer. --- checkstyle/checkstyle.xml | 4 +- checkstyle/import-control.xml | 1 + checkstyle/suppressions.xml | 8 +- .../clients/consumer/internals/Fetcher.java | 4 +- .../kafka/clients/producer/KafkaProducer.java | 111 ++- .../kafka/clients/producer/MockProducer.java | 67 ++ .../kafka/clients/producer/Producer.java | 59 +- .../clients/producer/ProducerConfig.java | 24 +- .../clients/producer/TransactionState.java | 642 +++++++++++++++++- .../internals/FutureTransactionalResult.java | 64 ++ .../producer/internals/RecordAccumulator.java | 16 +- .../clients/producer/internals/Sender.java | 94 ++- .../internals/TransactionalRequestResult.java | 64 ++ .../kafka/common/record/MemoryRecords.java | 20 +- .../common/record/MemoryRecordsBuilder.java | 27 +- .../requests/AddOffsetsToTxnResponse.java | 15 +- .../requests/AddPartitionsToTxnResponse.java | 1 + .../kafka/common/requests/EndTxnResponse.java | 1 + .../kafka/common/requests/ProduceRequest.java | 22 +- .../consumer/internals/FetcherTest.java | 6 +- .../producer/internals/TransactionsTest.java | 485 +++++++++++++ .../record/MemoryRecordsBuilderTest.java | 9 +- .../main/scala/kafka/log/LogValidator.scala | 4 +- 23 files changed, 1674 insertions(+), 74 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index 6a263ccc63eb8..27ef53b9ecfe4 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -102,7 +102,7 @@ - + @@ -115,7 +115,7 @@ - + diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 3475062105099..a6de9a727a8a6 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -124,6 +124,7 @@ + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 042be6b15bcac..a03d27f15546e 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -8,7 +8,7 @@ + files="(Fetcher|ConsumerCoordinator|KafkaConsumer|KafkaProducer|SaslServerAuthenticator|Utils|TransactionsTest).java"/> + + + files="(Sender|Fetcher|KafkaConsumer|Metrics|ConsumerCoordinator|RequestResponse|Transactions)Test.java"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index df96c3d792f4c..b1275ccad0ffc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -1046,10 +1046,10 @@ private boolean isBatchAborted(RecordBatch batch, Set abortedPids, Queue offsets, + String consumerGroupId) throws ProducerFencedException { + assert transactionState != null && transactionState.isInTransaction() && !transactionState.isCompletingTransaction() && transactionState.hasPid(); + FutureTransactionalResult result = transactionState.sendOffsetsToTransaction(offsets, consumerGroupId); + result.get(); + } + + /** + * Commits the ongoing transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + public void commitTransaction() throws ProducerFencedException { + assert transactionState != null && transactionState.isInTransaction() && transactionState.hasPid() && !transactionState.isCompletingTransaction(); + FutureTransactionalResult result = transactionState.beginCommittingTransaction(); + result.get(); + + } + + /** + * Aborts the ongoing transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + public void abortTransaction() throws ProducerFencedException { + assert transactionState != null && transactionState.isInTransaction() && transactionState.hasPid() && !transactionState.isCompletingTransaction(); + FutureTransactionalResult result = transactionState.beginAbortingTransaction(); + result.get(); + } + /** * Asynchronously send a record to a topic. Equivalent to send(record, null). * See {@link #send(ProducerRecord, Callback)} for details. @@ -523,6 +613,21 @@ public Future send(ProducerRecord record, Callback callbac * Implementation of asynchronously send a record to a topic. */ private Future doSend(ProducerRecord record, Callback callback) { + if (transactionState != null && transactionState.isTransactional() && !transactionState.hasPid()) { + if (transactionState.isFenced()) { + throw Errors.INVALID_PRODUCER_EPOCH.exception(); + } + throw new IllegalStateException("Cannot perform a 'send' before completing a call to initTransactions when transactions are enabled."); + } + + if (transactionState != null && transactionState.isCompletingTransaction()) { + throw new IllegalStateException("Cannot call send while a commit or abort is in progress."); + } + + if (transactionState != null && transactionState.isInTransaction()) { + transactionState.maybeAddPartitionToTransaction(new TopicPartition(record.topic(), record.partition())); + } + TopicPartition tp = null; try { // first make sure the metadata for the topic is available diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java index ab099977215a2..e753abcfcce27 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.clients.producer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.internals.DefaultPartitioner; import org.apache.kafka.clients.producer.internals.FutureRecordMetadata; import org.apache.kafka.clients.producer.internals.ProduceRequestResult; @@ -24,6 +25,7 @@ import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.ProducerFencedException; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.serialization.Serializer; @@ -95,6 +97,71 @@ public MockProducer(boolean autoComplete, Partitioner partitioner, Serializer this(Cluster.empty(), autoComplete, partitioner, keySerializer, valueSerializer); } + /** + * Needs to be called before any of the other transaction methods. Assumes that + * the transactional.id is specified in the producer configuration. + * + * This method does the following: + * 1. Ensures any transactions initiated by previous instances of the producer + * are completed. If the previous instance had failed with a transaction in + * progress, it will be aborted. If the last transaction had begun completion, + * but not yet finished, this method awaits its completion. + * 2. Gets the internal producer id and epoch, used in all future transactional + * messages issued by the producer. + * + * @throws IllegalStateException if the TransactionalId for the producer is not set + * in the configuration. + */ + public void initTransactions() { + + } + + /** + * Should be called before the start of each new transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + public void beginTransaction() throws ProducerFencedException { + + } + + /** + * Sends a list of consumed offsets to the consumer group coordinator, and also marks + * those offsets as part of the current transaction. These offsets will be considered + * consumed only if the transaction is committed successfully. + * + * This method should be used when you need to batch consumed and produced messages + * together, typically in a consume-transform-produce pattern. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + public void sendOffsetsToTransaction(Map offsets, + String consumerGroupId) throws ProducerFencedException { + + } + + /** + * Commits the ongoing transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + public void commitTransaction() throws ProducerFencedException { + + } + + /** + * Aborts the ongoing transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + public void abortTransaction() throws ProducerFencedException { + + } + /** * Adds the record to the list of sent records. The {@link RecordMetadata} returned will be immediately satisfied. * diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java index 4da868199fd18..9f7c617a5b1de 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java @@ -19,6 +19,9 @@ import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.ProducerFencedException; import java.io.Closeable; import java.util.List; @@ -26,7 +29,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; - /** * The interface for the {@link KafkaProducer} * @see KafkaProducer @@ -34,6 +36,61 @@ */ public interface Producer extends Closeable { + /** + * Needs to be called before any of the other transaction methods. Assumes that + * the transactional.id is specified in the producer configuration. + * + * This method does the following: + * 1. Ensures any transactions initiated by previous instances of the producer + * are completed. If the previous instance had failed with a transaction in + * progress, it will be aborted. If the last transaction had begun completion, + * but not yet finished, this method awaits its completion. + * 2. Gets the internal producer id and epoch, used in all future transactional + * messages issued by the producer. + * + * @throws IllegalStateException if the TransactionalId for the producer is not set + * in the configuration. + */ + void initTransactions() throws IllegalStateException; + + /** + * Should be called before the start of each new transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + void beginTransaction() throws ProducerFencedException; + + /** + * Sends a list of consumed offsets to the consumer group coordinator, and also marks + * those offsets as part of the current transaction. These offsets will be considered + * consumed only if the transaction is committed successfully. + * + * This method should be used when you need to batch consumed and produced messages + * together, typically in a consume-transform-produce pattern. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + void sendOffsetsToTransaction(Map offsets, + String consumerGroupId) throws ProducerFencedException; + + /** + * Commits the ongoing transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + void commitTransaction() throws ProducerFencedException; + + /** + * Aborts the ongoing transaction. + * + * @throws ProducerFencedException if another producer is with the same + * transactional.id is active. + */ + void abortTransaction() throws ProducerFencedException; + /** * Send the given record asynchronously and return a future which will eventually contain the response information. * diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index d6e03d287e9fc..afaa84a4a992f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -232,6 +232,18 @@ public class ProducerConfig extends AbstractConfig { + "" + RETRIES_CONFIG + " cannot be zero. Additionally " + ACKS_CONFIG + " must be set to 'all'. If these values " + "are left at their defaults, we will override the default to be suitable. " + "If the values are set to something incompatible with the idempotent producer, a ConfigException will be thrown."; + + /** transaction.timeout.ms */ + public static final String TRANSACTION_TIMEOUT_CONFIG = "transaction.timeout.ms"; + public static final String TRANSACTION_TIMEOUT_DOC = "The maximum amount of time in ms that the transaction coordinator will wait for a transaction status update from the producer before proactively aborting the ongoing transaction." + + "If this value is larger than the max.transaction.timeout.ms setting in the broker, the request will fail with a `InvalidTransactionTimeout` error."; + + /** transactional.id */ + public static final String TRANSACTIONAL_ID_CONFIG = "transactional.id"; + public static final String TRANSACTIONAL_ID_DOC = "The TransactionalId to use for transactional delivery. This enables reliability semantics which span multiple producer sessions since it allows the client to guarantee that transactions using the same TransactionalId have been completed prior to starting any new transactions. If no TransactionalId is provided, then the producer is limited to idempotent delivery. " + + "Note that enable.idempotence must be enabled if a TransactionalId is configured. " + + "The default is empty, which means transactions cannot be used."; + static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, Importance.HIGH, CommonClientConfigs.BOOTSTRAP_SERVERS_DOC) .define(BUFFER_MEMORY_CONFIG, Type.LONG, 32 * 1024 * 1024L, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) @@ -325,7 +337,17 @@ public class ProducerConfig extends AbstractConfig { Type.BOOLEAN, false, Importance.LOW, - ENABLE_IDEMPOTENCE_DOC); + ENABLE_IDEMPOTENCE_DOC) + .define(TRANSACTION_TIMEOUT_CONFIG, + Type.INT, + 60000, + Importance.LOW, + TRANSACTION_TIMEOUT_DOC) + .define(TRANSACTIONAL_ID_CONFIG, + Type.STRING, + "", + Importance.LOW, + TRANSACTIONAL_ID_DOC); } public static Map addSerializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index fa30b3f823421..66f8d11792a81 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -16,11 +16,41 @@ */ package org.apache.kafka.clients.producer; +import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.RequestCompletionHandler; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.internals.FutureTransactionalResult; +import org.apache.kafka.clients.producer.internals.TransactionalRequestResult; +import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.AddOffsetsToTxnRequest; +import org.apache.kafka.common.requests.AddOffsetsToTxnResponse; +import org.apache.kafka.common.requests.AddPartitionsToTxnRequest; +import org.apache.kafka.common.requests.AddPartitionsToTxnResponse; +import org.apache.kafka.common.requests.EndTxnRequest; +import org.apache.kafka.common.requests.EndTxnResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest; +import org.apache.kafka.common.requests.FindCoordinatorResponse; +import org.apache.kafka.common.requests.InitPidRequest; +import org.apache.kafka.common.requests.InitPidResponse; +import org.apache.kafka.common.requests.OffsetCommitRequest; +import org.apache.kafka.common.requests.TransactionResult; +import org.apache.kafka.common.requests.TxnOffsetCommitRequest; +import org.apache.kafka.common.requests.TxnOffsetCommitResponse; import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_EPOCH; import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_ID; @@ -29,9 +59,114 @@ * A class which maintains state for transactions. Also keeps the state necessary to ensure idempotent production. */ public class TransactionState { + private static final Logger log = LoggerFactory.getLogger(TransactionState.class); + private volatile PidAndEpoch pidAndEpoch; private final Map sequenceNumbers; - private final Time time; + private final String transactionalId; + private final int transactionTimeoutMs; + private final PriorityQueue pendingTransactionalRequests; + private final Set newPartitionsToBeAddedToTransaction; + private final Set pendingPartitionsToBeAddedToTransaction; + private final Set partitionsInTransaction; + private final Map pendingTxnOffsetCommits; + private int inFlightRequestCorrelationId = Integer.MIN_VALUE; + + private Node transactionCoordinator; + private Node consumerGroupCoordinator; + private volatile boolean isInTransaction = false; + private volatile boolean isCompletingTransaction = false; + private volatile boolean isInitializing = false; + private volatile boolean isFenced = false; + + public TransactionState(Time time, String transactionalId, int transactionTimeoutMs) { + pidAndEpoch = new PidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); + sequenceNumbers = new HashMap<>(); + this.transactionalId = transactionalId; + this.transactionTimeoutMs = transactionTimeoutMs; + this.pendingTransactionalRequests = new PriorityQueue<>(2, new Comparator() { + @Override + public int compare(TransactionalRequest o1, TransactionalRequest o2) { + return o1.priority().priority() - o2.priority.priority(); + } + }); + this.transactionCoordinator = null; + this.consumerGroupCoordinator = null; + this.newPartitionsToBeAddedToTransaction = new HashSet<>(); + this.pendingPartitionsToBeAddedToTransaction = new HashSet<>(); + this.partitionsInTransaction = new HashSet<>(); + this.pendingTxnOffsetCommits = new HashMap<>(); + } + + public static class TransactionalRequest { + private enum Priority { + FIND_COORDINATOR(0), + INIT_PRODUCER_ID(1), + ADD_PARTITIONS_OR_OFFSETS(2), + END_TXN(4); + + private final int priority; + + Priority(int priority) { + this.priority = priority; + } + + public int priority() { + return this.priority; + } + } + + private final AbstractRequest.Builder requestBuilder; + + private final FindCoordinatorRequest.CoordinatorType coordinatorType; + private final RequestCompletionHandler handler; + // We use the priority to determine the order in which requests need to be sent out. For instance, if we have + // a pending FindCoordinator request, that must always go first. Next, If we need a Pid, that must go second. + // The endTxn request must always go last. + private final Priority priority; + private boolean isRetry; + + private TransactionalRequest(AbstractRequest.Builder requestBuilder, RequestCompletionHandler handler, + FindCoordinatorRequest.CoordinatorType coordinatorType, Priority priority, boolean isRetry) { + this.requestBuilder = requestBuilder; + this.handler = handler; + this.coordinatorType = coordinatorType; + this.priority = priority; + this.isRetry = isRetry; + } + + public AbstractRequest.Builder requestBuilder() { + return requestBuilder; + } + + public FindCoordinatorRequest.CoordinatorType coordinatorType() { + return coordinatorType; + } + + public boolean needsCoordinator() { + return coordinatorType != null; + } + + public RequestCompletionHandler responseHandler() { + return handler; + } + + public boolean isRetry() { + return isRetry; + } + + public boolean isEndTxnRequest() { + return priority == Priority.END_TXN; + } + + private void setRetry() { + isRetry = true; + } + + private Priority priority() { + return priority; + } + } public static class PidAndEpoch { public final long producerId; @@ -47,32 +182,160 @@ public boolean isValid() { } } + public boolean hasPendingTransactionalRequests() { + return !(pendingTransactionalRequests.isEmpty() + && newPartitionsToBeAddedToTransaction.isEmpty()); + } + + + public TransactionalRequest nextTransactionalRequest() { + if (!hasPendingTransactionalRequests()) + return null; + + if (!newPartitionsToBeAddedToTransaction.isEmpty()) + pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false)); + + return pendingTransactionalRequests.poll(); + } + public TransactionState(Time time) { - this.pidAndEpoch = new PidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); - this.sequenceNumbers = new HashMap<>(); - this.time = time; + this(time, "", 0); + } + + public boolean isTransactional() { + return !transactionalId.isEmpty(); + } + + public String transactionalId() { + return transactionalId; } public boolean hasPid() { - return pidAndEpoch.isValid(); + return pidAndEpoch.isValid() && !isFenced; } - /** - * A blocking call to get the pid and epoch for the producer. If the PID and epoch has not been set, this method - * will block for at most maxWaitTimeMs. It is expected that this method be called from application thread - * contexts (ie. through Producer.send). The PID it self will be retrieved in the background thread. - * @param maxWaitTimeMs The maximum time to block. - * @return a PidAndEpoch object. Callers must call the 'isValid' method fo the returned object to ensure that a - * valid Pid and epoch is actually returned. - */ - public synchronized PidAndEpoch awaitPidAndEpoch(long maxWaitTimeMs) throws InterruptedException { - long start = time.milliseconds(); - long elapsed = 0; - while (!hasPid() && elapsed < maxWaitTimeMs) { - wait(maxWaitTimeMs); - elapsed = time.milliseconds() - start; + public boolean isFenced() { + return isFenced; + } + + public void beginTransaction() { + isInTransaction = true; + } + + public boolean isCompletingTransaction() { + return isInTransaction && isCompletingTransaction; + } + + public synchronized FutureTransactionalResult beginCommittingTransaction() { + return beginCompletingTransaction(true); + } + + public synchronized FutureTransactionalResult beginAbortingTransaction() { + return beginCompletingTransaction(false); + } + + private FutureTransactionalResult beginCompletingTransaction(boolean isCommit) { + if (!isCompletingTransaction) { + TransactionalRequestResult result = new TransactionalRequestResult(); + FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); + + isCompletingTransaction = true; + if (!newPartitionsToBeAddedToTransaction.isEmpty()) { + pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false)); + } + pendingTransactionalRequests.add(endTxnRequest(isCommit, false, result)); + return resultFuture; } - return pidAndEpoch; + return null; + } + + public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map offsets, + String consumerGroupId) { + TransactionalRequestResult result = new TransactionalRequestResult(); + FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); + pendingTransactionalRequests.add(addOffsetsToTxnRequest(offsets, consumerGroupId, false, result)); + return resultFuture; + } + + public boolean isInTransaction() { + return isTransactional() && isInTransaction; + } + + + public synchronized void maybeAddPartitionToTransaction(TopicPartition topicPartition) { + if (partitionsInTransaction.contains(topicPartition)) + return; + newPartitionsToBeAddedToTransaction.add(topicPartition); + } + + public void needsRetry(TransactionalRequest request) { + request.setRetry(); + pendingTransactionalRequests.add(request); + } + + public void didNotSend(TransactionalRequest request) { + pendingTransactionalRequests.add(request); + } + + public Node coordinator(FindCoordinatorRequest.CoordinatorType type) { + switch (type) { + case GROUP: + return consumerGroupCoordinator; + case TRANSACTION: + return transactionCoordinator; + default: + return null; + } + } + + public void needsCoordinator(FindCoordinatorRequest.CoordinatorType type) { + switch (type) { + case GROUP: + consumerGroupCoordinator = null; + case TRANSACTION: + transactionCoordinator = null; + } + pendingTransactionalRequests.add(findCoordinatorRequest(type, false)); + } + + public void setInFlightRequestCorrelationId(int correlationId) { + inFlightRequestCorrelationId = correlationId; + } + + public void resetInFlightRequestCorrelationId() { + inFlightRequestCorrelationId = Integer.MIN_VALUE; + } + + public boolean hasInflightTransactionalRequest() { + return inFlightRequestCorrelationId != Integer.MIN_VALUE; + } + + // visible for testing + public boolean transactionContainsPartition(TopicPartition topicPartition) { + return isInTransaction && partitionsInTransaction.contains(topicPartition); + } + + // visible for testing + public boolean hasPendingOffsetCommits() { + return isInTransaction && 0 < pendingTxnOffsetCommits.size(); + } + + public synchronized FutureTransactionalResult initializeTransactions() { + if (isInitializing) { + throw new IllegalStateException("Multiple concurrent calls to initTransactions are not allowed."); + } + isInitializing = true; + if (transactionCoordinator == null) + pendingTransactionalRequests.add(findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType.TRANSACTION, false)); + + TransactionalRequestResult result = new TransactionalRequestResult(); + FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); + if (!hasPid()) + pendingTransactionalRequests.add(initPidRequest(false, result)); + else + result.done(); + + return resultFuture; } /** @@ -91,8 +354,6 @@ public PidAndEpoch pidAndEpoch() { */ public synchronized void setPidAndEpoch(long pid, short epoch) { this.pidAndEpoch = new PidAndEpoch(pid, epoch); - if (this.pidAndEpoch.isValid()) - notifyAll(); } /** @@ -108,6 +369,15 @@ public synchronized void setPidAndEpoch(long pid, short epoch) { * messages will return an OutOfOrderSequenceException. */ public synchronized void resetProducerId() { + if (isTransactional()) { + // We can't reset the producer state for the transactional producer as this would mean bumping the epoch + // for the same pid. This might involve aborting the ongoing transaction during the initPidRequest, and + // the user would not have any way of knowing this happened. + // + // So it's best to return the produce error to the user and let them abort the transaction and close + // the producer explicitly. + return; + } setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); this.sequenceNumbers.clear(); } @@ -132,4 +402,332 @@ public synchronized void incrementSequenceNumber(TopicPartition topicPartition, currentSequenceNumber += increment; sequenceNumbers.put(topicPartition, currentSequenceNumber); } + + private void completeTransaction() { + partitionsInTransaction.clear(); + isInTransaction = false; + isCompletingTransaction = false; + } + + private TransactionalRequest initPidRequest(boolean isRetry, TransactionalRequestResult result) { + InitPidRequest.Builder builder = new InitPidRequest.Builder(transactionalId, transactionTimeoutMs); + return new TransactionalRequest(builder, new InitPidCallback(result), + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.INIT_PRODUCER_ID, isRetry); + } + + private synchronized TransactionalRequest addPartitionsToTransactionRequest(boolean isRetry) { + pendingPartitionsToBeAddedToTransaction.addAll(newPartitionsToBeAddedToTransaction); + newPartitionsToBeAddedToTransaction.clear(); + AddPartitionsToTxnRequest.Builder builder = new AddPartitionsToTxnRequest.Builder(transactionalId, + pidAndEpoch.producerId, pidAndEpoch.epoch, new ArrayList<>(pendingPartitionsToBeAddedToTransaction)); + return new TransactionalRequest(builder, new AddPartitionsToTransactionCallback(), + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry); + } + + private TransactionalRequest findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType type, boolean isRetry) { + FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder(type, transactionalId); + return new TransactionalRequest(builder, new FindCoordinatorCallback(type), + null, TransactionalRequest.Priority.FIND_COORDINATOR, isRetry); + } + + private TransactionalRequest endTxnRequest(boolean isCommit, boolean isRetry, TransactionalRequestResult result) { + EndTxnRequest.Builder builder = new EndTxnRequest.Builder(transactionalId, + pidAndEpoch.producerId, pidAndEpoch.epoch, isCommit ? TransactionResult.COMMIT : TransactionResult.ABORT); + return new TransactionalRequest(builder, new EndTxnCallback(isCommit, result), + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.END_TXN, isRetry); + } + + private TransactionalRequest addOffsetsToTxnRequest(Map offsets, + String consumerGroupId, boolean isRetry, TransactionalRequestResult result) { + AddOffsetsToTxnRequest.Builder builder = new AddOffsetsToTxnRequest.Builder(transactionalId, + pidAndEpoch.producerId, pidAndEpoch.epoch, consumerGroupId); + return new TransactionalRequest(builder, new AddOffsetsToTxnCallback(offsets, consumerGroupId, result), + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry); + } + + private TransactionalRequest txnOffsetCommitRequest(Map offsets, + String consumerGroupId, boolean isRetry, TransactionalRequestResult result) { + for (Map.Entry entry : offsets.entrySet()) { + OffsetAndMetadata offsetAndMetadata = entry.getValue(); + pendingTxnOffsetCommits.put(entry.getKey(), + new TxnOffsetCommitRequest.CommittedOffset(offsetAndMetadata.offset(), offsetAndMetadata.metadata())); + } + return txnOffsetCommitRequest(consumerGroupId, isRetry, result); + } + + private TransactionalRequest txnOffsetCommitRequest(String consumerGroupId, boolean isRetry, TransactionalRequestResult result) { + TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(consumerGroupId, + pidAndEpoch.producerId, pidAndEpoch.epoch, OffsetCommitRequest.DEFAULT_RETENTION_TIME, pendingTxnOffsetCommits); + return new TransactionalRequest(builder, new TxnOffsetCommitCallback(consumerGroupId, result), + FindCoordinatorRequest.CoordinatorType.GROUP, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry); + } + + private abstract class TransactionalRequestCallBack implements RequestCompletionHandler { + protected final TransactionalRequestResult result; + + TransactionalRequestCallBack(TransactionalRequestResult result) { + this.result = result; + } + + @Override + public void onComplete(ClientResponse response) { + if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) + throw new IllegalStateException("Cannot have more than one transactional request in flight."); + resetInFlightRequestCorrelationId(); + if (response.wasDisconnected()) { + reenqueue(); + } else if (response.versionMismatch() != null) { + if (result != null) { + result.setError(Errors.UNSUPPORTED_VERSION.exception()); + result.done(); + } else { + throw Errors.UNSUPPORTED_VERSION.exception(); + } + } else if (response.hasResponse()) { + handleResponse(response.responseBody()); + } else { + if (result != null) { + result.setError(Errors.UNKNOWN.exception()); + result.done(); + } else { + throw Errors.UNKNOWN.exception(); + } + } + } + + public abstract void handleResponse(AbstractResponse responseBody); + + public abstract void reenqueue(); + } + + private class InitPidCallback extends TransactionalRequestCallBack { + + InitPidCallback(TransactionalRequestResult result) { + super(result); + } + + @Override + public void handleResponse(AbstractResponse responseBody) { + InitPidResponse initPidResponse = (InitPidResponse) responseBody; + Errors error = initPidResponse.error(); + if (error == Errors.NONE) { + setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch()); + isInitializing = false; + } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) { + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + reenqueue(); + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + reenqueue(); + } else if (error == Errors.INVALID_TRANSACTION_TIMEOUT) { + result.setError(error.exception()); + } else { + result.setError(error.exception()); + } + + if (error == Errors.NONE || !result.isSuccessful()) + result.done(); + } + + @Override + public void reenqueue() { + pendingTransactionalRequests.add(initPidRequest(true, result)); + } + } + + private class AddPartitionsToTransactionCallback extends TransactionalRequestCallBack { + + AddPartitionsToTransactionCallback() { + super(null); + } + + @Override + public void handleResponse(AbstractResponse response) { + AddPartitionsToTxnResponse addPartitionsToTxnResponse = (AddPartitionsToTxnResponse) response; + Errors error = addPartitionsToTxnResponse.error(); + if (error == Errors.NONE) { + partitionsInTransaction.addAll(pendingPartitionsToBeAddedToTransaction); + pendingPartitionsToBeAddedToTransaction.clear(); + } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + reenqueue(); + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + reenqueue(); + } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { + throw error.exception(); + } else if (error == Errors.INVALID_PRODUCER_EPOCH) { + isFenced = true; + throw error.exception(); + } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) { + throw error.exception(); + } else { + throw Errors.UNKNOWN.exception(); + } + } + + @Override + public void reenqueue() { + pendingTransactionalRequests.add(addPartitionsToTransactionRequest(true)); + } + } + + private class FindCoordinatorCallback extends TransactionalRequestCallBack { + private final FindCoordinatorRequest.CoordinatorType type; + + FindCoordinatorCallback(FindCoordinatorRequest.CoordinatorType type) { + super(null); + this.type = type; + } + @Override + public void handleResponse(AbstractResponse responseBody) { + FindCoordinatorResponse findCoordinatorResponse = (FindCoordinatorResponse) responseBody; + if (findCoordinatorResponse.error() == Errors.NONE) { + Node node = findCoordinatorResponse.node(); + switch (type) { + case GROUP: + consumerGroupCoordinator = node; + case TRANSACTION: + transactionCoordinator = node; + } + } else if (findCoordinatorResponse.error() == Errors.COORDINATOR_NOT_AVAILABLE) { + reenqueue(); + } else if (findCoordinatorResponse.error() == Errors.GROUP_AUTHORIZATION_FAILED) { + throw Errors.GROUP_AUTHORIZATION_FAILED.exception(); + } else { + throw Errors.UNKNOWN.exception(); + } + } + + @Override + public void reenqueue() { + pendingTransactionalRequests.add(findCoordinatorRequest(type, true)); + } + } + + private class EndTxnCallback extends TransactionalRequestCallBack { + private final boolean isCommit; + + EndTxnCallback(boolean isCommit, TransactionalRequestResult result) { + super(result); + this.isCommit = isCommit; + } + + @Override + public void handleResponse(AbstractResponse responseBody) { + EndTxnResponse endTxnResponse = (EndTxnResponse) responseBody; + Errors error = endTxnResponse.error(); + if (error == Errors.NONE) { + completeTransaction(); + } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + reenqueue(); + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + reenqueue(); + } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { + result.setError(error.exception()); + } else if (error == Errors.INVALID_PRODUCER_EPOCH) { + isFenced = true; + result.setError(error.exception()); + } else { + result.setError(error.exception()); + } + + if (error == Errors.NONE || !result.isSuccessful()) + result.done(); + } + + @Override + public void reenqueue() { + pendingTransactionalRequests.add(endTxnRequest(isCommit, true, result)); + } + } + + private class AddOffsetsToTxnCallback extends TransactionalRequestCallBack { + String consumerGroupId; + Map offsets; + + AddOffsetsToTxnCallback(Map offsets, String consumerGroupId, TransactionalRequestResult result) { + super(result); + this.offsets = offsets; + this.consumerGroupId = consumerGroupId; + } + + @Override + public void handleResponse(AbstractResponse responseBody) { + AddOffsetsToTxnResponse addOffsetsToTxnResponse = (AddOffsetsToTxnResponse) responseBody; + Errors error = addOffsetsToTxnResponse.error(); + if (error == Errors.NONE) { + consumerGroupCoordinator = addOffsetsToTxnResponse.consumerGroupCoordinator(); + pendingTransactionalRequests.add(txnOffsetCommitRequest(offsets, consumerGroupId, false, result)); + } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + reenqueue(); + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + reenqueue(); + } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { + result.setError(error.exception()); + } else if (error == Errors.INVALID_PRODUCER_EPOCH) { + isFenced = true; + result.setError(error.exception()); + } else { + result.setError(error.exception()); + } + + if (!result.isSuccessful()) + result.done(); + } + + @Override + public void reenqueue() { + pendingTransactionalRequests.add(addOffsetsToTxnRequest(offsets, consumerGroupId, true, result)); + } + } + + private class TxnOffsetCommitCallback extends TransactionalRequestCallBack { + private final String consumerGroupId; + + TxnOffsetCommitCallback(String consumerGroupId, TransactionalRequestResult result) { + super(result); + this.consumerGroupId = consumerGroupId; + } + + @Override + public void handleResponse(AbstractResponse responseBody) { + TxnOffsetCommitResponse txnOffsetCommitResponse = (TxnOffsetCommitResponse) responseBody; + boolean coordinatorReloaded = false; + boolean hadFailure = false; + for (Map.Entry entry : txnOffsetCommitResponse.errors().entrySet()) { + TopicPartition topicPartition = entry.getKey(); + Errors error = entry.getValue(); + if (error == Errors.NONE) { + pendingTxnOffsetCommits.remove(topicPartition); + } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { + hadFailure = true; + if (!coordinatorReloaded) { + coordinatorReloaded = true; + needsCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP); + } + } else if (error == Errors.INVALID_PRODUCER_EPOCH) { + isFenced = true; + result.setError(error.exception()); + break; + } + } + + if (!hadFailure || !result.isSuccessful()) { + // all attempted partitions were either successful, or there was a fatal failure. + // either way, we are not retrying, so complete the request. + result.done(); + return; + } + + if (0 < pendingTxnOffsetCommits.size()) { + pendingTransactionalRequests.add(txnOffsetCommitRequest(consumerGroupId, true, result)); + } + } + + @Override + public void reenqueue() { + pendingTransactionalRequests.add(txnOffsetCommitRequest(consumerGroupId, true, result)); + } + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java new file mode 100644 index 0000000000000..d05bc6a32d813 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.producer.internals; + +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public final class FutureTransactionalResult implements Future { + + private final TransactionalRequestResult result; + + public FutureTransactionalResult(TransactionalRequestResult result) { + this.result = result; + } + + @Override + public boolean isDone() { + return this.result.isCompleted(); + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public TransactionalRequestResult get() { + this.result.await(); + if (!result.isSuccessful()) { + throw result.error(); + } + return result; + } + + @Override + public TransactionalRequestResult get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { + boolean occurred = this.result.await(timeout, unit); + if (!occurred) { + throw new TimeoutException("Could not complete transactional operation within " + TimeUnit.MILLISECONDS.convert(timeout, unit) + "ms."); + } + return result; + } + +} diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index be0314271caa0..efc983faea246 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -233,7 +233,10 @@ private MemoryRecordsBuilder recordsBuilder(ByteBuffer buffer, byte maxUsableMag throw new UnsupportedVersionException("Attempting to use idempotence with a broker which does not " + "support the required message format (v2). The broker must be version 0.11 or later."); } - return MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L); + boolean isTransactional = false; + if (transactionState != null) + isTransactional = transactionState.isInTransaction(); + return MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L, isTransactional); } /** @@ -542,6 +545,17 @@ public void awaitFlushCompletion() throws InterruptedException { } } + public boolean hasUnflushedBatches() { + boolean hasUnflushed = false; + for (Map.Entry> entry : this.batches().entrySet()) { + if (0 < entry.getValue().size()) { + hasUnflushed = true; + break; + } + } + return 0 < this.incomplete.incomplete.size() || hasUnflushed; + } + /** * This function is only called when sender is closed forcefully. It will fail all the * incomplete batches and return. diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index ab92522a04c1f..6b58066b2681c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -187,8 +187,16 @@ public void run() { * @param now The current POSIX time in milliseconds */ void run(long now) { - Cluster cluster = metadata.fetch(); + long pollTimeout = 0; + if (!maybeSendTransactionalRequest(now)) + pollTimeout = sendProducerData(now); + + this.client.poll(pollTimeout, now); + } + + private long sendProducerData(long now) { + Cluster cluster = metadata.fetch(); maybeWaitForPid(); // get the list of partitions with data ready to send @@ -241,7 +249,7 @@ void run(long now) { if (needsTransactionStateReset) { transactionState.resetProducerId(); - return; + return 0; } sensors.updateProduceRequestMetrics(batches); @@ -253,15 +261,77 @@ void run(long now) { long pollTimeout = Math.min(result.nextReadyCheckDelayMs, notReadyTimeout); if (!result.readyNodes.isEmpty()) { log.trace("Nodes with data ready to send: {}", result.readyNodes); + // if some partitions are already ready to be sent, the select time would be 0; + // otherwise if some partition already has some data accumulated but not ready yet, + // the select time will be the time difference between now and its linger expiry time; + // otherwise the select time will be the time difference between now and the metadata expiry time; pollTimeout = 0; } sendProduceRequests(batches, now); - // if some partitions are already ready to be sent, the select time would be 0; - // otherwise if some partition already has some data accumulated but not ready yet, - // the select time will be the time difference between now and its linger expiry time; - // otherwise the select time will be the time difference between now and the metadata expiry time; - this.client.poll(pollTimeout, now); + return pollTimeout; + + } + + private boolean maybeSendTransactionalRequest(long now) { + if (transactionState != null && transactionState.hasInflightTransactionalRequest()) + return true; + + if (transactionState == null || !transactionState.hasPendingTransactionalRequests()) + return false; + + TransactionState.TransactionalRequest nextRequest = transactionState.nextTransactionalRequest(); + + if (nextRequest.isEndTxnRequest() && transactionState.isCompletingTransaction() && accumulator.hasUnflushedBatches()) { + if (!accumulator.flushInProgress()) + accumulator.beginFlush(); + transactionState.didNotSend(nextRequest); + return false; + } + + Node targetNode = null; + long expiryTime = now + requestTimeout; + long nextIterationTime = now; + + while (targetNode == null && nextIterationTime < expiryTime) { + try { + long timeRemaining = expiryTime - nextIterationTime; + if (nextRequest.needsCoordinator()) { + targetNode = transactionState.coordinator(nextRequest.coordinatorType()); + if (targetNode == null) { + transactionState.needsCoordinator(nextRequest.coordinatorType()); + break; + } + if (!NetworkClientUtils.awaitReady(client, targetNode, time, timeRemaining)) { + transactionState.needsCoordinator(nextRequest.coordinatorType()); + targetNode = null; + break; + } + } else { + targetNode = awaitLeastLoadedNodeReady(timeRemaining); + } + if (targetNode != null) { + if (nextRequest.isRetry()) { + time.sleep(retryBackoffMs); + } + ClientRequest clientRequest = client.newClientRequest(targetNode.idString(), nextRequest.requestBuilder(), + now, true, nextRequest.responseHandler()); + transactionState.setInFlightRequestCorrelationId(clientRequest.correlationId()); + client.send(clientRequest, now); + return true; + } + } catch (IOException e) { + log.warn("Got an exception when trying to find a node to send a transactional request to. Going to back off and retry", e); + } + time.sleep(retryBackoffMs); + metadata.requestUpdate(); + nextIterationTime = time.milliseconds(); + } + + if (targetNode == null) + transactionState.needsRetry(nextRequest); + + return true; } /** @@ -299,7 +369,9 @@ private Node awaitLeastLoadedNodeReady(long remainingTimeMs) throws IOException } private void maybeWaitForPid() { - if (transactionState == null) + // If this is a transactional producer, the PID will be received when recovering transactions in the + // initTransactions() method of the producer. + if (transactionState == null || transactionState.isTransactional()) return; while (!transactionState.hasPid()) { @@ -500,8 +572,12 @@ private void sendProduceRequest(long now, int destination, short acks, int timeo recordsByPartition.put(tp, batch); } + String transactionalId = null; + if (transactionState != null && transactionState.isTransactional()) { + transactionalId = transactionState.transactionalId(); + } ProduceRequest.Builder requestBuilder = new ProduceRequest.Builder(minUsedMagic, acks, timeout, - produceRecordsByPartition); + produceRecordsByPartition, transactionalId); RequestCompletionHandler callback = new RequestCompletionHandler() { public void onComplete(ClientResponse response) { handleProduceResponse(response, recordsByPartition, time.milliseconds()); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java new file mode 100644 index 0000000000000..88f146bff00fb --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.producer.internals; + + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public final class TransactionalRequestResult { + private final CountDownLatch latch = new CountDownLatch(1); + private RuntimeException error = null; + + public void setError(RuntimeException error) { + this.error = error; + } + + public void done() { + this.latch.countDown(); + } + + public void await() { + boolean completed = false; + + while (!completed) { + try { + latch.await(); + completed = true; + } catch (InterruptedException e) { + // Keep waiting until done, we have no other option for these transactional requests. + } + } + } + + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { + return latch.await(timeout, unit); + } + + public RuntimeException error() { + return error; + } + + public boolean isSuccessful() { + return error == null; + } + + public boolean isCompleted() { + return latch.getCount() == 0L; + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java index 5c32bd59a04e0..16588c2302863 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java @@ -297,10 +297,7 @@ public static MemoryRecordsBuilder builder(ByteBuffer buffer, CompressionType compressionType, TimestampType timestampType, long baseOffset) { - long logAppendTime = RecordBatch.NO_TIMESTAMP; - if (timestampType == TimestampType.LOG_APPEND_TIME) - logAppendTime = System.currentTimeMillis(); - return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime); + return builder(buffer, magic, compressionType, timestampType, baseOffset, false); } public static MemoryRecordsBuilder builder(ByteBuffer buffer, @@ -313,6 +310,19 @@ public static MemoryRecordsBuilder builder(ByteBuffer buffer, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, RecordBatch.NO_PARTITION_LEADER_EPOCH); } + public static MemoryRecordsBuilder builder(ByteBuffer buffer, + byte magic, + CompressionType compressionType, + TimestampType timestampType, + long baseOffset, + boolean isTransactional) { + long logAppendTime = RecordBatch.NO_TIMESTAMP; + if (timestampType == TimestampType.LOG_APPEND_TIME) + logAppendTime = System.currentTimeMillis(); + return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime, + RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, isTransactional, RecordBatch.NO_PARTITION_LEADER_EPOCH); + } + public static MemoryRecordsBuilder builder(ByteBuffer buffer, byte magic, CompressionType compressionType, @@ -400,7 +410,7 @@ private static MemoryRecords withRecords(byte magic, long initialOffset, Compres if (timestampType == TimestampType.LOG_APPEND_TIME) logAppendTime = System.currentTimeMillis(); MemoryRecordsBuilder builder = builder(buffer, magic, compressionType, timestampType, initialOffset, - logAppendTime, producerId, producerEpoch, baseSequence, partitionLeaderEpoch); + logAppendTime, producerId, producerEpoch, baseSequence, false, partitionLeaderEpoch); for (SimpleRecord record : records) builder.append(record); return builder.build(); diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 549804a291b16..935f0e6b6c60c 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -117,23 +117,10 @@ public MemoryRecordsBuilder(ByteBuffer buffer, throw new IllegalArgumentException("TimestampType must be set for magic >= 0"); if (isTransactional) { - if (producerId == RecordBatch.NO_PRODUCER_ID) - throw new IllegalArgumentException("Cannot write transactional messages without a valid producer ID"); - if (magic < RecordBatch.MAGIC_VALUE_V2) throw new IllegalArgumentException("Transactional messages are not supported for magic " + magic); } - if (producerId != RecordBatch.NO_PRODUCER_ID) { - if (producerEpoch < 0) - throw new IllegalArgumentException("Invalid negative producer epoch"); - - if (baseSequence < 0) - throw new IllegalArgumentException("Invalid negative sequence number used"); - - if (magic < RecordBatch.MAGIC_VALUE_V2) - throw new IllegalArgumentException("Idempotent messages are not supported for magic " + magic); - } this.magic = magic; this.timestampType = timestampType; @@ -261,6 +248,20 @@ public void close() { closeForRecordAppends(); + if (isTransactional && producerId == RecordBatch.NO_PRODUCER_ID) + throw new IllegalArgumentException("Cannot write transactional messages without a valid producer ID"); + + if (producerId != RecordBatch.NO_PRODUCER_ID) { + if (producerEpoch == RecordBatch.NO_PRODUCER_EPOCH) + throw new IllegalArgumentException("Invalid negative producer epoch"); + + if (baseSequence == RecordBatch.NO_SEQUENCE) + throw new IllegalArgumentException("Invalid negative sequence number used"); + + if (magic < RecordBatch.MAGIC_VALUE_V2) + throw new IllegalArgumentException("Idempotent messages are not supported for magic " + magic); + } + if (numRecords == 0L) { buffer().position(initPos); builtRecords = MemoryRecords.EMPTY; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java index 6ac49fb89efe1..c62a96e0a9415 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.common.requests; +import org.apache.kafka.common.Node; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -32,21 +33,33 @@ public class AddOffsetsToTxnResponse extends AbstractResponse { // InvalidPidMapping // InvalidTxnState // GroupAuthorizationFailed + // InvalidProducerEpoch private final Errors error; + private final Node consumerGroupCoordinator; - public AddOffsetsToTxnResponse(Errors error) { + public AddOffsetsToTxnResponse(Errors error, Node consumerGroupCoordinator) { this.error = error; + this.consumerGroupCoordinator = consumerGroupCoordinator; + } + + public AddOffsetsToTxnResponse(Errors error) { + this(error, null); } public AddOffsetsToTxnResponse(Struct struct) { this.error = Errors.forCode(struct.getShort(ERROR_CODE_KEY_NAME)); + this.consumerGroupCoordinator = null; } public Errors error() { return error; } + public Node consumerGroupCoordinator() { + return consumerGroupCoordinator; + } + @Override protected Struct toStruct(short version) { Struct struct = new Struct(ApiKeys.ADD_OFFSETS_TO_TXN.responseSchema(version)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java index 3de6295b56f0d..03370445e3f8e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java @@ -32,6 +32,7 @@ public class AddPartitionsToTxnResponse extends AbstractResponse { // InvalidTxnState // InvalidPidMapping // TopicAuthorizationFailed + // InvalidProducerEpoch private final Errors error; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java index 627eb6422f0c3..3ff6aca2783d4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java @@ -31,6 +31,7 @@ public class EndTxnResponse extends AbstractResponse { // CoordinatorLoadInProgress // InvalidTxnState // InvalidPidMapping + // InvalidProducerEpoch private final Errors error; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index 76313910dc887..8265e04690500 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -55,16 +55,26 @@ public static class Builder extends AbstractRequest.Builder { private final short acks; private final int timeout; private final Map partitionRecords; + private final String transactioanlId; public Builder(byte magic, short acks, int timeout, - Map partitionRecords) { + Map partitionRecords, + String transactionalId) { super(ApiKeys.PRODUCE, (short) (magic == RecordBatch.MAGIC_VALUE_V2 ? 3 : 2)); this.magic = magic; this.acks = acks; this.timeout = timeout; this.partitionRecords = partitionRecords; + this.transactioanlId = transactionalId; + } + + public Builder(byte magic, + short acks, + int timeout, + Map partitionRecords) { + this(magic, acks, timeout, partitionRecords, null); } @Override @@ -72,7 +82,7 @@ public ProduceRequest build(short version) { if (version < 2) throw new UnsupportedVersionException("ProduceRequest versions older than 2 are not supported."); - return new ProduceRequest(version, acks, timeout, partitionRecords); + return new ProduceRequest(version, acks, timeout, partitionRecords, transactioanlId); } @Override @@ -83,6 +93,7 @@ public String toString() { .append(", acks=").append(acks) .append(", timeout=").append(timeout) .append(", partitionRecords=(").append(partitionRecords) + .append(", transactionalId=(").append(transactioanlId != null ? transactioanlId : "") .append("))"); return bld.toString(); } @@ -100,12 +111,15 @@ public String toString() { private volatile Map partitionRecords; private ProduceRequest(short version, short acks, int timeout, Map partitionRecords) { + this(version, acks, timeout, partitionRecords, null); + } + + private ProduceRequest(short version, short acks, int timeout, Map partitionRecords, String transactionalId) { super(version); this.acks = acks; this.timeout = timeout; - // TODO: Include transactional id in constructor once transactions are supported - this.transactionalId = null; + this.transactionalId = transactionalId; this.partitionRecords = partitionRecords; this.partitionSizes = createPartitionSizes(partitionRecords); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 911abcf5ca78f..2aa6336c05aa4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1334,7 +1334,7 @@ public void testReturnAbortedTransactionsinUncommittedMode() { private int appendTransactionalRecords(ByteBuffer buffer, long pid, int baseOffset, SimpleRecord... records) { MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, - TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true); + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); for (SimpleRecord record : records) { builder.append(record); @@ -1345,7 +1345,7 @@ private int appendTransactionalRecords(ByteBuffer buffer, long pid, int baseOffs private int commitTransaction(ByteBuffer buffer, long pid, int baseOffset, long timestamp) { MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, - TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true); + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); builder.appendControlRecord(timestamp, ControlRecordType.COMMIT, null); builder.build(); return 1; @@ -1353,7 +1353,7 @@ private int commitTransaction(ByteBuffer buffer, long pid, int baseOffset, long private int abortTransaction(ByteBuffer buffer, long pid, int baseOffset, long timestamp) { MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, - TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true); + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); builder.appendControlRecord(timestamp, ControlRecordType.ABORT, null); builder.build(); return 1; diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java new file mode 100644 index 0000000000000..0d6bdc7957fca --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java @@ -0,0 +1,485 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.producer.internals; + +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.MockClient; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.clients.producer.TransactionState; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MutableRecordBatch; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AddOffsetsToTxnRequest; +import org.apache.kafka.common.requests.AddOffsetsToTxnResponse; +import org.apache.kafka.common.requests.AddPartitionsToTxnRequest; +import org.apache.kafka.common.requests.AddPartitionsToTxnResponse; +import org.apache.kafka.common.requests.EndTxnRequest; +import org.apache.kafka.common.requests.EndTxnResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest; +import org.apache.kafka.common.requests.FindCoordinatorResponse; +import org.apache.kafka.common.requests.InitPidRequest; +import org.apache.kafka.common.requests.InitPidResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.TransactionResult; +import org.apache.kafka.common.requests.TxnOffsetCommitRequest; +import org.apache.kafka.common.requests.TxnOffsetCommitResponse; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.test.TestUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TransactionsTest { + private static final int MAX_REQUEST_SIZE = 1024 * 1024; + private static final short ACKS_ALL = -1; + private static final int MAX_RETRIES = 0; + private static final String CLIENT_ID = "clientId"; + private static final int MAX_BLOCK_TIMEOUT = 1000; + private static final int REQUEST_TIMEOUT = 1000; + private final String transactionalId = "foobar"; + private final int transactionTimeoutMs = 1121; + + private TopicPartition tp0 = new TopicPartition("test", 0); + private TopicPartition tp1 = new TopicPartition("test", 1); + private MockTime time = new MockTime(); + private MockClient client = new MockClient(time); + + private Metadata metadata = new Metadata(0, Long.MAX_VALUE, true, new ClusterResourceListeners()); + private ApiVersions apiVersions = new ApiVersions(); + private Cluster cluster = TestUtils.singletonCluster("test", 2); + private RecordAccumulator accumulator = null; + private Sender sender = null; + private TransactionState transactionState = null; + private Node brokerNode = null; + + @Before + public void setup() { + Map metricTags = new LinkedHashMap<>(); + metricTags.put("client-id", CLIENT_ID); + int batchSize = 16 * 1024; + MetricConfig metricConfig = new MetricConfig().tags(metricTags); + this.brokerNode = new Node(0, "localhost", 2211); + this.transactionState = new TransactionState(time, transactionalId, transactionTimeoutMs); + Metrics metrics = new Metrics(metricConfig, time); + this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionState); + this.sender = new Sender(this.client, + this.metadata, + this.accumulator, + true, + MAX_REQUEST_SIZE, + ACKS_ALL, + MAX_RETRIES, + metrics, + this.time, + REQUEST_TIMEOUT, + 50, + transactionState, + apiVersions); + this.metadata.update(this.cluster, Collections.emptySet(), time.milliseconds()); + } + + + @Test + public void testBasicTransaction() throws InterruptedException { + client.setNode(brokerNode); + // This is called from the initTransactions method in the producer as the first order of business. + // It finds the coordinator and then gets a PID. + final long pid = 13131L; + final short epoch = 1; + transactionState.initializeTransactions(); + prepareFindCoordinatorResponse(Errors.NONE, false); + + sender.run(time.milliseconds()); // find coordinator + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + + prepareInitPidResponse(Errors.NONE, false, pid, epoch); + + sender.run(time.milliseconds()); // get pid. + + assertTrue(transactionState.hasPid()); + transactionState.beginTransaction(); + transactionState.maybeAddPartitionToTransaction(tp0); + + Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + + assertFalse(responseFuture.isDone()); + prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); + + prepareProduceResponse(Errors.NONE, pid, epoch); + assertFalse(transactionState.transactionContainsPartition(tp0)); + sender.run(time.milliseconds()); // send addPartitions. + // Check that only addPartitions was sent. + assertTrue(transactionState.transactionContainsPartition(tp0)); + assertFalse(responseFuture.isDone()); + + sender.run(time.milliseconds()); // send produce request. + assertTrue(responseFuture.isDone()); + + Map offsets = new HashMap<>(); + offsets.put(tp1, new OffsetAndMetadata(1)); + final String consumerGroupId = "myconsumergroup"; + FutureTransactionalResult addOffsetsResult = transactionState.sendOffsetsToTransaction(offsets, consumerGroupId); + + assertFalse(transactionState.hasPendingOffsetCommits()); + + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + AddOffsetsToTxnRequest addOffsetsToTxnRequest = (AddOffsetsToTxnRequest) body; + assertEquals(consumerGroupId, addOffsetsToTxnRequest.consumerGroupId()); + assertEquals(transactionalId, addOffsetsToTxnRequest.transactionalId()); + assertEquals(pid, addOffsetsToTxnRequest.producerId()); + assertEquals(epoch, addOffsetsToTxnRequest.producerEpoch()); + return true; + } + }, new AddOffsetsToTxnResponse(Errors.NONE, brokerNode)); + + sender.run(time.milliseconds()); // Send AddOffsetsRequest + assertTrue(transactionState.hasPendingOffsetCommits()); // We should now have created and queued the offset commit request. + assertFalse(addOffsetsResult.isDone()); + + Map txnOffsetCommitResponse = new HashMap<>(); + txnOffsetCommitResponse.put(tp1, Errors.NONE); + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + TxnOffsetCommitRequest txnOffsetCommitRequest = (TxnOffsetCommitRequest) body; + assertEquals(consumerGroupId, txnOffsetCommitRequest.consumerGroupId()); + assertEquals(pid, txnOffsetCommitRequest.producerId()); + assertEquals(epoch, txnOffsetCommitRequest.producerEpoch()); + return true; + } + }, new TxnOffsetCommitResponse(txnOffsetCommitResponse)); + + + sender.run(time.milliseconds()); // send offset commit. + assertFalse(transactionState.hasPendingOffsetCommits()); + assertTrue(addOffsetsResult.isDone()); // We should only be done after both RPCs complete. + + transactionState.beginCommittingTransaction(); + prepareEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch); + sender.run(time.milliseconds()); // commit. + + assertFalse(transactionState.isInTransaction()); + assertFalse(transactionState.isCompletingTransaction()); + assertFalse(transactionState.transactionContainsPartition(tp0)); + } + + @Test + public void testDisconnectAndRetry() { + client.setNode(brokerNode); + // This is called from the initTransactions method in the producer as the first order of business. + // It finds the coordinator and then gets a PID. + transactionState.initializeTransactions(); + prepareFindCoordinatorResponse(Errors.NONE, true); + sender.run(time.milliseconds()); // find coordinator, connection lost. + + prepareFindCoordinatorResponse(Errors.NONE, false); + sender.run(time.milliseconds()); // find coordinator + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + } + + @Test + public void testCoordinatorLost() { + client.setNode(brokerNode); + // This is called from the initTransactions method in the producer as the first order of business. + // It finds the coordinator and then gets a PID. + final long pid = 13131L; + final short epoch = 1; + FutureTransactionalResult initPidResult = transactionState.initializeTransactions(); + prepareFindCoordinatorResponse(Errors.NONE, false); + sender.run(time.milliseconds()); // find coordinator + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + + prepareInitPidResponse(Errors.NOT_COORDINATOR, false, pid, epoch); + sender.run(time.milliseconds()); // send pid, get not coordinator. Should resend the FindCoordinator and InitPid requests + + assertEquals(null, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertFalse(initPidResult.isDone()); + assertFalse(transactionState.hasPid()); + + prepareFindCoordinatorResponse(Errors.NONE, false); + sender.run(time.milliseconds()); + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertFalse(initPidResult.isDone()); + prepareInitPidResponse(Errors.NONE, false, pid, epoch); + sender.run(time.milliseconds()); // get pid and epoch + + assertTrue(initPidResult.isDone()); // The future should only return after the second round of retries succeed. + assertTrue(transactionState.hasPid()); + assertEquals(pid, transactionState.pidAndEpoch().producerId); + assertEquals(epoch, transactionState.pidAndEpoch().epoch); + } + + @Test + public void testFlushPendingPartitionsOnCommit() throws InterruptedException { + client.setNode(brokerNode); + // This is called from the initTransactions method in the producer as the first order of business. + // It finds the coordinator and then gets a PID. + final long pid = 13131L; + final short epoch = 1; + transactionState.initializeTransactions(); + prepareFindCoordinatorResponse(Errors.NONE, false); + + sender.run(time.milliseconds()); // find coordinator + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + + prepareInitPidResponse(Errors.NONE, false, pid, epoch); + + sender.run(time.milliseconds()); // get pid. + + assertTrue(transactionState.hasPid()); + + transactionState.beginTransaction(); + transactionState.maybeAddPartitionToTransaction(tp0); + + Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + + assertFalse(responseFuture.isDone()); + + FutureTransactionalResult commitResult = transactionState.beginCommittingTransaction(); + + // we have an append, an add partitions request, and now also an endtxn. + // The order should be: + // 1. Add Partitions + // 2. Produce + // 3. EndTxn. + assertFalse(transactionState.transactionContainsPartition(tp0)); + prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); + + sender.run(time.milliseconds()); // AddPartitions. + assertTrue(transactionState.transactionContainsPartition(tp0)); + assertFalse(responseFuture.isDone()); + assertFalse(commitResult.isDone()); + + prepareProduceResponse(Errors.NONE, pid, epoch); + sender.run(time.milliseconds()); // Produce. + assertTrue(responseFuture.isDone()); + + prepareEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch); + assertFalse(commitResult.isDone()); + assertTrue(transactionState.isInTransaction()); + assertTrue(transactionState.isCompletingTransaction()); + + sender.run(time.milliseconds()); + assertTrue(commitResult.isDone()); + assertFalse(transactionState.isInTransaction()); + } + + @Test + public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedException { + client.setNode(brokerNode); + // This is called from the initTransactions method in the producer as the first order of business. + // It finds the coordinator and then gets a PID. + final long pid = 13131L; + final short epoch = 1; + transactionState.initializeTransactions(); + prepareFindCoordinatorResponse(Errors.NONE, false); + + sender.run(time.milliseconds()); // find coordinator + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + + prepareInitPidResponse(Errors.NONE, false, pid, epoch); + + sender.run(time.milliseconds()); // get pid. + + assertTrue(transactionState.hasPid()); + transactionState.beginTransaction(); + // User does one producer.sed + transactionState.maybeAddPartitionToTransaction(tp0); + + Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + + assertFalse(responseFuture.isDone()); + prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); + + assertFalse(transactionState.transactionContainsPartition(tp0)); + + // Sender flushes one add partitions. The produce goes next. + sender.run(time.milliseconds()); // send addPartitions. + // Check that only addPartitions was sent. + assertTrue(transactionState.transactionContainsPartition(tp0)); + + // In the mean time, the user does a second produce to a different partition + transactionState.maybeAddPartitionToTransaction(tp1); + Future secondResponseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + + prepareAddPartitionsToTxnResponse(Errors.NONE, tp1, epoch, pid); + prepareProduceResponse(Errors.NONE, pid, epoch); + + assertFalse(transactionState.transactionContainsPartition(tp1)); + + assertFalse(responseFuture.isDone()); + assertFalse(secondResponseFuture.isDone()); + + // The second add partitionsh should go out here. + sender.run(time.milliseconds()); // send second add partitions request + assertTrue(transactionState.transactionContainsPartition(tp1)); + + assertFalse(responseFuture.isDone()); + assertFalse(secondResponseFuture.isDone()); + + // Finally we get to the produce. + sender.run(time.milliseconds()); // send produce request + + assertTrue(responseFuture.isDone()); + assertTrue(secondResponseFuture.isDone()); + } + + @Test(expected = ExecutionException.class) + public void testProducerFencedException() throws InterruptedException, ExecutionException { + client.setNode(brokerNode); + // This is called from the initTransactions method in the producer as the first order of business. + // It finds the coordinator and then gets a PID. + final long pid = 13131L; + final short epoch = 1; + transactionState.initializeTransactions(); + prepareFindCoordinatorResponse(Errors.NONE, false); + + sender.run(time.milliseconds()); // find coordinator + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + + prepareInitPidResponse(Errors.NONE, false, pid, epoch); + + sender.run(time.milliseconds()); // get pid. + + assertTrue(transactionState.hasPid()); + transactionState.beginTransaction(); + transactionState.maybeAddPartitionToTransaction(tp0); + + Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; + + assertFalse(responseFuture.isDone()); + prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); + prepareProduceResponse(Errors.INVALID_PRODUCER_EPOCH, pid, epoch); + sender.run(time.milliseconds()); // Add partitions. + + sender.run(time.milliseconds()); // send produce. + + responseFuture.get(); + } + + private void prepareFindCoordinatorResponse(Errors error, boolean shouldDisconnect) { + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + FindCoordinatorRequest findCoordinatorRequest = (FindCoordinatorRequest) body; + assertEquals(findCoordinatorRequest.coordinatorType(), FindCoordinatorRequest.CoordinatorType.TRANSACTION); + assertEquals(findCoordinatorRequest.coordinatorKey(), transactionalId); + return true; + } + }, new FindCoordinatorResponse(error, brokerNode), shouldDisconnect); + } + + private void prepareInitPidResponse(Errors error, boolean shouldDisconnect, long pid, short epoch) { + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + InitPidRequest initPidRequest = (InitPidRequest) body; + assertEquals(initPidRequest.transactionalId(), transactionalId); + assertEquals(initPidRequest.transactionTimeoutMs(), transactionTimeoutMs); + return true; + } + }, new InitPidResponse(error, pid, epoch), shouldDisconnect); + } + + private void prepareProduceResponse(Errors error, final long pid, final short epoch) { + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + ProduceRequest produceRequest = (ProduceRequest) body; + MemoryRecords records = produceRequest.partitionRecordsOrFail().get(tp0); + assertNotNull(records); + Iterator batchIterator = records.batches().iterator(); + assertTrue(batchIterator.hasNext()); + MutableRecordBatch batch = batchIterator.next(); + assertFalse(batchIterator.hasNext()); + assertTrue(batch.isTransactional()); + assertEquals(pid, batch.producerId()); + assertEquals(epoch, batch.producerEpoch()); + assertEquals(transactionalId, produceRequest.transactionalId()); + return true; + } + }, produceResponse(tp0, 0, error, 0)); + + } + + private void prepareAddPartitionsToTxnResponse(Errors error, final TopicPartition topicPartition, final short epoch, final long pid) { + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + AddPartitionsToTxnRequest addPartitionsToTxnRequest = (AddPartitionsToTxnRequest) body; + assertEquals(pid, addPartitionsToTxnRequest.producerId()); + assertEquals(epoch, addPartitionsToTxnRequest.producerEpoch()); + assertEquals(Arrays.asList(topicPartition), addPartitionsToTxnRequest.partitions()); + assertEquals(transactionalId, addPartitionsToTxnRequest.transactionalId()); + return true; + } + }, new AddPartitionsToTxnResponse(error)); + } + + private void prepareEndTxnResponse(Errors error, final TransactionResult result, final long pid, final short epoch) { + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + EndTxnRequest endTxnRequest = (EndTxnRequest) body; + assertEquals(transactionalId, endTxnRequest.transactionalId()); + assertEquals(pid, endTxnRequest.producerId()); + assertEquals(epoch, endTxnRequest.producerEpoch()); + assertEquals(result, endTxnRequest.command()); + return true; + } + }, new EndTxnResponse(error)); + } + + private ProduceResponse produceResponse(TopicPartition tp, long offset, Errors error, int throttleTimeMs) { + ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse(error, offset, RecordBatch.NO_TIMESTAMP); + Map partResp = Collections.singletonMap(tp, resp); + return new ProduceResponse(partResp, throttleTimeMs); + } + +} diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java index 3cc8c20d199ad..330879fa2b2b9 100644 --- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java @@ -111,8 +111,9 @@ public void testWriteTransactionalWithInvalidPID() { short epoch = 15; int sequence = 2342; - new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + builder.close(); } @Test(expected = IllegalArgumentException.class) @@ -124,8 +125,9 @@ public void testWriteIdempotentWithInvalidEpoch() { short epoch = RecordBatch.NO_PRODUCER_EPOCH; int sequence = 2342; - new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + builder.close(); } @Test(expected = IllegalArgumentException.class) @@ -137,8 +139,9 @@ public void testWriteIdempotentWithInvalidBaseSequence() { short epoch = 15; int sequence = RecordBatch.NO_SEQUENCE; - new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + builder.close(); } @Test diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala index 0616d41ab3094..c1777d529dcd9 100644 --- a/core/src/main/scala/kafka/log/LogValidator.scala +++ b/core/src/main/scala/kafka/log/LogValidator.scala @@ -107,7 +107,7 @@ private[kafka] object LogValidator extends Logging { val newBuffer = ByteBuffer.allocate(sizeInBytesAfterConversion) val builder = MemoryRecords.builder(newBuffer, toMagicValue, CompressionType.NONE, timestampType, - offsetCounter.value, now, pid, epoch, sequence, partitionLeaderEpoch) + offsetCounter.value, now, pid, epoch, sequence, false, partitionLeaderEpoch) for (batch <- records.batches.asScala) { validateBatch(batch) @@ -281,7 +281,7 @@ private[kafka] object LogValidator extends Logging { val estimatedSize = AbstractRecords.estimateSizeInBytes(magic, offsetCounter.value, compressionType, validatedRecords.asJava) val buffer = ByteBuffer.allocate(estimatedSize) val builder = MemoryRecords.builder(buffer, magic, compressionType, timestampType, offsetCounter.value, - logAppendTime, producerId, epoch, baseSequence, partitionLeaderEpoch) + logAppendTime, producerId, epoch, baseSequence, false, partitionLeaderEpoch) validatedRecords.foreach { record => builder.appendWithOffset(offsetCounter.getAndIncrement(), record) From ec3944bd13bfa0555087329c320177a981104d85 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 11 Apr 2017 12:09:50 -0700 Subject: [PATCH 03/21] Drop the consumerCoordinator field from the AddOffsetsToTxn response. Implemented the ability to find the group coordinator from the producer. --- .../clients/producer/TransactionState.java | 60 +++++++++++-------- .../clients/producer/internals/Sender.java | 4 +- .../requests/AddOffsetsToTxnResponse.java | 14 +---- .../producer/internals/TransactionsTest.java | 37 ++++++++---- 4 files changed, 62 insertions(+), 53 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index 66f8d11792a81..9c5640a590d1f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -119,6 +119,7 @@ public int priority() { private final AbstractRequest.Builder requestBuilder; private final FindCoordinatorRequest.CoordinatorType coordinatorType; + private final String coordinatorKey; private final RequestCompletionHandler handler; // We use the priority to determine the order in which requests need to be sent out. For instance, if we have // a pending FindCoordinator request, that must always go first. Next, If we need a Pid, that must go second. @@ -127,26 +128,28 @@ public int priority() { private boolean isRetry; private TransactionalRequest(AbstractRequest.Builder requestBuilder, RequestCompletionHandler handler, - FindCoordinatorRequest.CoordinatorType coordinatorType, Priority priority, boolean isRetry) { + FindCoordinatorRequest.CoordinatorType coordinatorType, Priority priority, + boolean isRetry, String coordinatorKey) { this.requestBuilder = requestBuilder; this.handler = handler; this.coordinatorType = coordinatorType; this.priority = priority; this.isRetry = isRetry; + this.coordinatorKey = coordinatorKey; } public AbstractRequest.Builder requestBuilder() { return requestBuilder; } - public FindCoordinatorRequest.CoordinatorType coordinatorType() { - return coordinatorType; - } - public boolean needsCoordinator() { return coordinatorType != null; } + public FindCoordinatorRequest.CoordinatorType coordinatorType() { + return coordinatorType; + } + public RequestCompletionHandler responseHandler() { return handler; } @@ -288,16 +291,22 @@ public Node coordinator(FindCoordinatorRequest.CoordinatorType type) { } } - public void needsCoordinator(FindCoordinatorRequest.CoordinatorType type) { + public void needsCoordinator(TransactionalRequest request) { + needsCoordinator(request.coordinatorType, request.coordinatorKey); + } + + private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) { switch (type) { case GROUP: consumerGroupCoordinator = null; + break; case TRANSACTION: transactionCoordinator = null; } - pendingTransactionalRequests.add(findCoordinatorRequest(type, false)); + pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, false)); } + public void setInFlightRequestCorrelationId(int correlationId) { inFlightRequestCorrelationId = correlationId; } @@ -326,7 +335,7 @@ public synchronized FutureTransactionalResult initializeTransactions() { } isInitializing = true; if (transactionCoordinator == null) - pendingTransactionalRequests.add(findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType.TRANSACTION, false)); + pendingTransactionalRequests.add(findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId, false)); TransactionalRequestResult result = new TransactionalRequestResult(); FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); @@ -412,7 +421,7 @@ private void completeTransaction() { private TransactionalRequest initPidRequest(boolean isRetry, TransactionalRequestResult result) { InitPidRequest.Builder builder = new InitPidRequest.Builder(transactionalId, transactionTimeoutMs); return new TransactionalRequest(builder, new InitPidCallback(result), - FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.INIT_PRODUCER_ID, isRetry); + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.INIT_PRODUCER_ID, isRetry, transactionalId); } private synchronized TransactionalRequest addPartitionsToTransactionRequest(boolean isRetry) { @@ -421,20 +430,20 @@ private synchronized TransactionalRequest addPartitionsToTransactionRequest(bool AddPartitionsToTxnRequest.Builder builder = new AddPartitionsToTxnRequest.Builder(transactionalId, pidAndEpoch.producerId, pidAndEpoch.epoch, new ArrayList<>(pendingPartitionsToBeAddedToTransaction)); return new TransactionalRequest(builder, new AddPartitionsToTransactionCallback(), - FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry); + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, transactionalId); } - private TransactionalRequest findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType type, boolean isRetry) { - FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder(type, transactionalId); - return new TransactionalRequest(builder, new FindCoordinatorCallback(type), - null, TransactionalRequest.Priority.FIND_COORDINATOR, isRetry); + private TransactionalRequest findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey, boolean isRetry) { + FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder(type, coordinatorKey); + return new TransactionalRequest(builder, new FindCoordinatorCallback(type, coordinatorKey), + null, TransactionalRequest.Priority.FIND_COORDINATOR, isRetry, null); } private TransactionalRequest endTxnRequest(boolean isCommit, boolean isRetry, TransactionalRequestResult result) { EndTxnRequest.Builder builder = new EndTxnRequest.Builder(transactionalId, pidAndEpoch.producerId, pidAndEpoch.epoch, isCommit ? TransactionResult.COMMIT : TransactionResult.ABORT); return new TransactionalRequest(builder, new EndTxnCallback(isCommit, result), - FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.END_TXN, isRetry); + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.END_TXN, isRetry, transactionalId); } private TransactionalRequest addOffsetsToTxnRequest(Map offsets, @@ -442,7 +451,7 @@ private TransactionalRequest addOffsetsToTxnRequest(Map offsets, @@ -459,7 +468,7 @@ private TransactionalRequest txnOffsetCommitRequest(String consumerGroupId, bool TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(consumerGroupId, pidAndEpoch.producerId, pidAndEpoch.epoch, OffsetCommitRequest.DEFAULT_RETENTION_TIME, pendingTxnOffsetCommits); return new TransactionalRequest(builder, new TxnOffsetCommitCallback(consumerGroupId, result), - FindCoordinatorRequest.CoordinatorType.GROUP, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry); + FindCoordinatorRequest.CoordinatorType.GROUP, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, consumerGroupId); } private abstract class TransactionalRequestCallBack implements RequestCompletionHandler { @@ -514,7 +523,7 @@ public void handleResponse(AbstractResponse responseBody) { setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch()); isInitializing = false; } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) { - needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); @@ -548,7 +557,7 @@ public void handleResponse(AbstractResponse response) { partitionsInTransaction.addAll(pendingPartitionsToBeAddedToTransaction); pendingPartitionsToBeAddedToTransaction.clear(); } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { - needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); @@ -572,10 +581,12 @@ public void reenqueue() { private class FindCoordinatorCallback extends TransactionalRequestCallBack { private final FindCoordinatorRequest.CoordinatorType type; + private final String coordinatorKey; - FindCoordinatorCallback(FindCoordinatorRequest.CoordinatorType type) { + FindCoordinatorCallback(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) { super(null); this.type = type; + this.coordinatorKey = coordinatorKey; } @Override public void handleResponse(AbstractResponse responseBody) { @@ -599,7 +610,7 @@ public void handleResponse(AbstractResponse responseBody) { @Override public void reenqueue() { - pendingTransactionalRequests.add(findCoordinatorRequest(type, true)); + pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, true)); } } @@ -618,7 +629,7 @@ public void handleResponse(AbstractResponse responseBody) { if (error == Errors.NONE) { completeTransaction(); } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { - needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); @@ -656,10 +667,9 @@ public void handleResponse(AbstractResponse responseBody) { AddOffsetsToTxnResponse addOffsetsToTxnResponse = (AddOffsetsToTxnResponse) responseBody; Errors error = addOffsetsToTxnResponse.error(); if (error == Errors.NONE) { - consumerGroupCoordinator = addOffsetsToTxnResponse.consumerGroupCoordinator(); pendingTransactionalRequests.add(txnOffsetCommitRequest(offsets, consumerGroupId, false, result)); } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { - needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION); + needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); @@ -704,7 +714,7 @@ public void handleResponse(AbstractResponse responseBody) { hadFailure = true; if (!coordinatorReloaded) { coordinatorReloaded = true; - needsCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP); + needsCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP, consumerGroupId); } } else if (error == Errors.INVALID_PRODUCER_EPOCH) { isFenced = true; diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 6b58066b2681c..12d585c6e68b4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -299,11 +299,11 @@ private boolean maybeSendTransactionalRequest(long now) { if (nextRequest.needsCoordinator()) { targetNode = transactionState.coordinator(nextRequest.coordinatorType()); if (targetNode == null) { - transactionState.needsCoordinator(nextRequest.coordinatorType()); + transactionState.needsCoordinator(nextRequest); break; } if (!NetworkClientUtils.awaitReady(client, targetNode, time, timeRemaining)) { - transactionState.needsCoordinator(nextRequest.coordinatorType()); + transactionState.needsCoordinator(nextRequest); targetNode = null; break; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java index c62a96e0a9415..2426514dfb00f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.common.requests; -import org.apache.kafka.common.Node; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -36,30 +35,19 @@ public class AddOffsetsToTxnResponse extends AbstractResponse { // InvalidProducerEpoch private final Errors error; - private final Node consumerGroupCoordinator; - - public AddOffsetsToTxnResponse(Errors error, Node consumerGroupCoordinator) { - this.error = error; - this.consumerGroupCoordinator = consumerGroupCoordinator; - } public AddOffsetsToTxnResponse(Errors error) { - this(error, null); + this.error = error; } public AddOffsetsToTxnResponse(Struct struct) { this.error = Errors.forCode(struct.getShort(ERROR_CODE_KEY_NAME)); - this.consumerGroupCoordinator = null; } public Errors error() { return error; } - public Node consumerGroupCoordinator() { - return consumerGroupCoordinator; - } - @Override protected Struct toStruct(short version) { Struct struct = new Struct(ApiKeys.ADD_OFFSETS_TO_TXN.responseSchema(version)); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java index 0d6bdc7957fca..0a88b8df3d11e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java @@ -126,7 +126,7 @@ public void testBasicTransaction() throws InterruptedException { final long pid = 13131L; final short epoch = 1; transactionState.initializeTransactions(); - prepareFindCoordinatorResponse(Errors.NONE, false); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); @@ -172,7 +172,7 @@ public boolean matches(AbstractRequest body) { assertEquals(epoch, addOffsetsToTxnRequest.producerEpoch()); return true; } - }, new AddOffsetsToTxnResponse(Errors.NONE, brokerNode)); + }, new AddOffsetsToTxnResponse(Errors.NONE)); sender.run(time.milliseconds()); // Send AddOffsetsRequest assertTrue(transactionState.hasPendingOffsetCommits()); // We should now have created and queued the offset commit request. @@ -180,6 +180,9 @@ public boolean matches(AbstractRequest body) { Map txnOffsetCommitResponse = new HashMap<>(); txnOffsetCommitResponse.put(tp1, Errors.NONE); + + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.GROUP, consumerGroupId); + client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { @@ -191,8 +194,14 @@ public boolean matches(AbstractRequest body) { } }, new TxnOffsetCommitResponse(txnOffsetCommitResponse)); + assertEquals(null, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP)); + sender.run(time.milliseconds()); // try to send TxnOffsetCommitRequest, but find we don't have a group coordinator. + sender.run(time.milliseconds()); // send find coordinator for group request + assertNotNull(transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP)); + assertTrue(transactionState.hasPendingOffsetCommits()); + + sender.run(time.milliseconds()); // send TxnOffsetCommitRequest commit. - sender.run(time.milliseconds()); // send offset commit. assertFalse(transactionState.hasPendingOffsetCommits()); assertTrue(addOffsetsResult.isDone()); // We should only be done after both RPCs complete. @@ -211,10 +220,10 @@ public void testDisconnectAndRetry() { // This is called from the initTransactions method in the producer as the first order of business. // It finds the coordinator and then gets a PID. transactionState.initializeTransactions(); - prepareFindCoordinatorResponse(Errors.NONE, true); + prepareFindCoordinatorResponse(Errors.NONE, true, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator, connection lost. - prepareFindCoordinatorResponse(Errors.NONE, false); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); } @@ -227,7 +236,7 @@ public void testCoordinatorLost() { final long pid = 13131L; final short epoch = 1; FutureTransactionalResult initPidResult = transactionState.initializeTransactions(); - prepareFindCoordinatorResponse(Errors.NONE, false); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); @@ -238,7 +247,7 @@ public void testCoordinatorLost() { assertFalse(initPidResult.isDone()); assertFalse(transactionState.hasPid()); - prepareFindCoordinatorResponse(Errors.NONE, false); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); assertFalse(initPidResult.isDone()); @@ -259,7 +268,7 @@ public void testFlushPendingPartitionsOnCommit() throws InterruptedException { final long pid = 13131L; final short epoch = 1; transactionState.initializeTransactions(); - prepareFindCoordinatorResponse(Errors.NONE, false); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); @@ -315,7 +324,7 @@ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedExcept final long pid = 13131L; final short epoch = 1; transactionState.initializeTransactions(); - prepareFindCoordinatorResponse(Errors.NONE, false); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); @@ -377,7 +386,7 @@ public void testProducerFencedException() throws InterruptedException, Execution final long pid = 13131L; final short epoch = 1; transactionState.initializeTransactions(); - prepareFindCoordinatorResponse(Errors.NONE, false); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); @@ -403,13 +412,15 @@ public void testProducerFencedException() throws InterruptedException, Execution responseFuture.get(); } - private void prepareFindCoordinatorResponse(Errors error, boolean shouldDisconnect) { + private void prepareFindCoordinatorResponse(Errors error, boolean shouldDisconnect, + final FindCoordinatorRequest.CoordinatorType coordinatorType, + final String coordinatorKey) { client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { FindCoordinatorRequest findCoordinatorRequest = (FindCoordinatorRequest) body; - assertEquals(findCoordinatorRequest.coordinatorType(), FindCoordinatorRequest.CoordinatorType.TRANSACTION); - assertEquals(findCoordinatorRequest.coordinatorKey(), transactionalId); + assertEquals(findCoordinatorRequest.coordinatorType(), coordinatorType); + assertEquals(findCoordinatorRequest.coordinatorKey(), coordinatorKey); return true; } }, new FindCoordinatorResponse(error, brokerNode), shouldDisconnect); From cd20e88b35a6609331203d5fe9fb9699e5ed688f Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 11 Apr 2017 13:11:32 -0700 Subject: [PATCH 04/21] Add break in FindCoordinatorCallback --- .../java/org/apache/kafka/clients/producer/TransactionState.java | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index 9c5640a590d1f..e17d6041d7a96 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -596,6 +596,7 @@ public void handleResponse(AbstractResponse responseBody) { switch (type) { case GROUP: consumerGroupCoordinator = node; + break; case TRANSACTION: transactionCoordinator = node; } From c7eba46c1a081d94015259c25c2a578044973132 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 13 Apr 2017 14:57:01 -0700 Subject: [PATCH 05/21] Addressed PR comments --- checkstyle/checkstyle.xml | 4 +- checkstyle/suppressions.xml | 44 ++----------------- .../clients/consumer/ConsumerConfig.java | 8 ++-- .../kafka/clients/consumer/KafkaConsumer.java | 11 ++--- .../clients/consumer/internals/Fetcher.java | 9 ++-- .../kafka/clients/producer/KafkaProducer.java | 29 ++++++++---- .../kafka/clients/producer/MockProducer.java | 44 ------------------- .../kafka/clients/producer/Producer.java | 8 ++-- .../clients/producer/ProducerConfig.java | 2 +- .../clients/producer/TransactionState.java | 41 +++++++++-------- .../producer/internals/RecordAccumulator.java | 9 ++-- .../common/record/MemoryRecordsBuilder.java | 4 +- .../kafka/common/requests/ProduceRequest.java | 14 +++--- .../clients/consumer/KafkaConsumerTest.java | 4 +- .../internals/RecordAccumulatorTest.java | 2 +- .../producer/internals/SenderTest.java | 8 ++-- .../internals/TransactionStateTest.java | 7 ++- .../producer/internals/TransactionsTest.java | 2 +- .../unit/kafka/server/FetchRequestTest.scala | 3 +- 19 files changed, 90 insertions(+), 163 deletions(-) diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index 27ef53b9ecfe4..6a263ccc63eb8 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -102,7 +102,7 @@ - + @@ -115,7 +115,7 @@ - + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index a03d27f15546e..b90b7948a774b 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -35,7 +35,7 @@ files="DefaultRecordBatch.java"/> + files="(KafkaConsumer|ConsumerCoordinator|Fetcher|KafkaProducer|AbstractRequest|AbstractResponse|TransactionState).java"/> @@ -43,51 +43,13 @@ files="(Utils|KafkaLZ4BlockOutputStream).java"/> - - - - - - - - - - - + files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|SsLTransportLayer|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslTransportLayer).java"/> - - - - - - - - + files="(BufferPool|MetricName|Node|ConfigDef|SslTransportLayer|MetadataResponse|KerberosLogin|SslTransportLayer|Sender).java"/> READ_UNCOMMITTED' (the default), consumer.poll() will return all messages, even transactional messages" + - " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.

Messages will be always returned in offset order. Hence, in " + + " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.

Messages will always be returned in offset order. Hence, in " + " READ_COMMITTED mode, consumer.poll() will only return messages upto the last resolved (committed or aborted) transaction. In particular any messages appearing after" + - " messages belonging onging transactions will be withheld until the said transaction has been completed and its messages are delivered to the application. As a result, READ_COMMITTED" + - " consumers will not be able to read upto the log end offset when there are inflight transactions.

"; + " messages belonging to onging transactions will be withheld until the relevant transaction has been completed. As a result, READ_COMMITTED" + + " consumers will not be able to read up to the log end offset when there are inflight transactions.

"; public static final String DEFAULT_ISOLATION_LEVEL = IsolationLevel.READ_UNCOMMITTED.toString(); @@ -420,7 +420,7 @@ public class ConsumerConfig extends AbstractConfig { .define(ISOLATION_LEVEL_CONFIG, Type.STRING, DEFAULT_ISOLATION_LEVEL, - Importance.LOW, + Importance.MEDIUM, ISOLATION_LEVEL_DOC) // security support .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index fba0ae1c143d0..da5b7fbb840c8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -533,7 +533,6 @@ public class KafkaConsumer implements Consumer { private final Metadata metadata; private final long retryBackoffMs; private final long requestTimeoutMs; - private final IsolationLevel isolationLevel; private volatile boolean closed = false; // currentThread holds the threadId of the current thread accessing KafkaConsumer @@ -663,8 +662,8 @@ private KafkaConsumer(ConsumerConfig config, String metricGrpPrefix = "consumer"; ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); - this.isolationLevel = IsolationLevel.valueOf( - config.getString(ConsumerConfig.ISOLATION_LEVEL_CONFIG).toUpperCase(Locale.ENGLISH)); + IsolationLevel isolationLevel = IsolationLevel.valueOf( + config.getString(ConsumerConfig.ISOLATION_LEVEL_CONFIG).toUpperCase(Locale.ROOT)); NetworkClient netClient = new NetworkClient( new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), @@ -717,7 +716,7 @@ private KafkaConsumer(ConsumerConfig config, metricGrpPrefix, this.time, this.retryBackoffMs, - IsolationLevel.READ_UNCOMMITTED); + isolationLevel); config.logUnused(); AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); @@ -745,8 +744,7 @@ private KafkaConsumer(ConsumerConfig config, SubscriptionState subscriptions, Metadata metadata, long retryBackoffMs, - long requestTimeoutMs, - String isolationLevel) { + long requestTimeoutMs) { this.clientId = clientId; this.coordinator = coordinator; this.keyDeserializer = keyDeserializer; @@ -760,7 +758,6 @@ private KafkaConsumer(ConsumerConfig config, this.metadata = metadata; this.retryBackoffMs = retryBackoffMs; this.requestTimeoutMs = requestTimeoutMs; - this.isolationLevel = IsolationLevel.valueOf(isolationLevel.toUpperCase(Locale.ENGLISH)); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index b1275ccad0ffc..4327337c7a4e2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -382,7 +382,7 @@ private long offsetResetStrategyTimestamp(final TopicPartition partition) { if (strategy == OffsetResetStrategy.EARLIEST) timestamp = ListOffsetRequest.EARLIEST_TIMESTAMP; else if (strategy == OffsetResetStrategy.LATEST) - timestamp = isolationLevel == IsolationLevel.READ_COMMITTED ? ListOffsetRequest.LSO_TIMESTAMP : ListOffsetRequest.LATEST_TIMESTAMP; + timestamp = endTimestamp(); else throw new NoOffsetForPartitionException(partition); return timestamp; @@ -469,8 +469,11 @@ public Map beginningOffsets(Collection par } public Map endOffsets(Collection partitions, long timeout) { - long endTimestamp = isolationLevel == IsolationLevel.READ_COMMITTED ? ListOffsetRequest.LSO_TIMESTAMP : ListOffsetRequest.LATEST_TIMESTAMP; - return beginningOrEndOffset(partitions, endTimestamp, timeout); + return beginningOrEndOffset(partitions, endTimestamp(), timeout); + } + + private long endTimestamp() { + return isolationLevel == IsolationLevel.READ_COMMITTED ? ListOffsetRequest.LSO_TIMESTAMP : ListOffsetRequest.LATEST_TIMESTAMP; } private Map beginningOrEndOffset(Collection partitions, diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 39564a9ebda25..5669a47994e01 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -373,7 +373,7 @@ private static TransactionState configureTransactionState(ProducerConfig config, if (idempotenceEnabled) { String transactionalId = config.getString(ProducerConfig.TRANSACTIONAL_ID_CONFIG); int transactionTimeoutMs = config.getInt(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG); - transactionState = new TransactionState(time, transactionalId, transactionTimeoutMs); + transactionState = new TransactionState(transactionalId, transactionTimeoutMs); if (transactionState.isTransactional()) log.info("Instantiated a transactional producer."); else @@ -473,7 +473,10 @@ public void initTransactions() { */ public void beginTransaction() throws ProducerFencedException { // Set the transactional bit in the producer. - assert transactionState != null && transactionState.isTransactional() && !transactionState.isInTransaction() && transactionState.hasPid(); + if (transactionState == null || !transactionState.isTransactional() || transactionState.isInTransaction() + || transactionState.isCompletingTransaction() || !transactionState.hasPid()) + throw new IllegalStateException("Cannot begin a new transaction. Either transactions are not configured, " + + "or there is already a transaction ongoing, or the producer is fenced."); transactionState.beginTransaction(); } @@ -485,12 +488,16 @@ public void beginTransaction() throws ProducerFencedException { * This method should be used when you need to batch consumed and produced messages * together, typically in a consume-transform-produce pattern. * - * @throws ProducerFencedException if another producer is with the same + * @throws ProducerFencedException if another producer with the same * transactional.id is active. */ public void sendOffsetsToTransaction(Map offsets, String consumerGroupId) throws ProducerFencedException { - assert transactionState != null && transactionState.isInTransaction() && !transactionState.isCompletingTransaction() && transactionState.hasPid(); + if (transactionState == null || !transactionState.isInTransaction() || transactionState.isCompletingTransaction() + || !transactionState.hasPid()) + throw new IllegalStateException("Cannot send offsets to transaction. Either transactions are not enabled, or a transaction" + + " isn't active right now, or a transaction is in the process of being completed, or the producer" + + " has been fenced."); FutureTransactionalResult result = transactionState.sendOffsetsToTransaction(offsets, consumerGroupId); result.get(); } @@ -498,11 +505,14 @@ public void sendOffsetsToTransaction(Map offs /** * Commits the ongoing transaction. * - * @throws ProducerFencedException if another producer is with the same + * @throws ProducerFencedException if another producer with the same * transactional.id is active. */ public void commitTransaction() throws ProducerFencedException { - assert transactionState != null && transactionState.isInTransaction() && transactionState.hasPid() && !transactionState.isCompletingTransaction(); + if (transactionState == null || !transactionState.isInTransaction() || !transactionState.hasPid() + || transactionState.isCompletingTransaction()) + throw new IllegalStateException("Cannot commit transaction. Either transactions are not enabled, or there is" + + " no transaction in progress, or the producer is fenced, or a transaction is already completing."); FutureTransactionalResult result = transactionState.beginCommittingTransaction(); result.get(); @@ -511,11 +521,14 @@ public void commitTransaction() throws ProducerFencedException { /** * Aborts the ongoing transaction. * - * @throws ProducerFencedException if another producer is with the same + * @throws ProducerFencedException if another producer with the same * transactional.id is active. */ public void abortTransaction() throws ProducerFencedException { - assert transactionState != null && transactionState.isInTransaction() && transactionState.hasPid() && !transactionState.isCompletingTransaction(); + if (transactionState == null || !transactionState.isInTransaction() || !transactionState.hasPid() + || transactionState.isCompletingTransaction()) + throw new IllegalStateException("Cannot commit transaction. Either transactions are not enabled, or there is" + + " no transaction in progress, or the producer is fenced, or a transaction is already completing."); FutureTransactionalResult result = transactionState.beginAbortingTransaction(); result.get(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java index e753abcfcce27..80ea372531f26 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java @@ -97,67 +97,23 @@ public MockProducer(boolean autoComplete, Partitioner partitioner, Serializer this(Cluster.empty(), autoComplete, partitioner, keySerializer, valueSerializer); } - /** - * Needs to be called before any of the other transaction methods. Assumes that - * the transactional.id is specified in the producer configuration. - * - * This method does the following: - * 1. Ensures any transactions initiated by previous instances of the producer - * are completed. If the previous instance had failed with a transaction in - * progress, it will be aborted. If the last transaction had begun completion, - * but not yet finished, this method awaits its completion. - * 2. Gets the internal producer id and epoch, used in all future transactional - * messages issued by the producer. - * - * @throws IllegalStateException if the TransactionalId for the producer is not set - * in the configuration. - */ public void initTransactions() { } - /** - * Should be called before the start of each new transaction. - * - * @throws ProducerFencedException if another producer is with the same - * transactional.id is active. - */ public void beginTransaction() throws ProducerFencedException { } - /** - * Sends a list of consumed offsets to the consumer group coordinator, and also marks - * those offsets as part of the current transaction. These offsets will be considered - * consumed only if the transaction is committed successfully. - * - * This method should be used when you need to batch consumed and produced messages - * together, typically in a consume-transform-produce pattern. - * - * @throws ProducerFencedException if another producer is with the same - * transactional.id is active. - */ public void sendOffsetsToTransaction(Map offsets, String consumerGroupId) throws ProducerFencedException { } - /** - * Commits the ongoing transaction. - * - * @throws ProducerFencedException if another producer is with the same - * transactional.id is active. - */ public void commitTransaction() throws ProducerFencedException { } - /** - * Aborts the ongoing transaction. - * - * @throws ProducerFencedException if another producer is with the same - * transactional.id is active. - */ public void abortTransaction() throws ProducerFencedException { } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java index 9f7c617a5b1de..0554c9232d5b1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java @@ -56,7 +56,7 @@ public interface Producer extends Closeable { /** * Should be called before the start of each new transaction. * - * @throws ProducerFencedException if another producer is with the same + * @throws ProducerFencedException if another producer with the same * transactional.id is active. */ void beginTransaction() throws ProducerFencedException; @@ -69,7 +69,7 @@ public interface Producer extends Closeable { * This method should be used when you need to batch consumed and produced messages * together, typically in a consume-transform-produce pattern. * - * @throws ProducerFencedException if another producer is with the same + * @throws ProducerFencedException if another producer with the same * transactional.id is active. */ void sendOffsetsToTransaction(Map offsets, @@ -78,7 +78,7 @@ void sendOffsetsToTransaction(Map offsets, /** * Commits the ongoing transaction. * - * @throws ProducerFencedException if another producer is with the same + * @throws ProducerFencedException if another producer with the same * transactional.id is active. */ void commitTransaction() throws ProducerFencedException; @@ -86,7 +86,7 @@ void sendOffsetsToTransaction(Map offsets, /** * Aborts the ongoing transaction. * - * @throws ProducerFencedException if another producer is with the same + * @throws ProducerFencedException if another producer with the same * transactional.id is active. */ void abortTransaction() throws ProducerFencedException; diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index afaa84a4a992f..b54b589ec9371 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -345,7 +345,7 @@ public class ProducerConfig extends AbstractConfig { TRANSACTION_TIMEOUT_DOC) .define(TRANSACTIONAL_ID_CONFIG, Type.STRING, - "", + null, Importance.LOW, TRANSACTIONAL_ID_DOC); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index e17d6041d7a96..964c3e6c3bd4a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -40,7 +40,6 @@ import org.apache.kafka.common.requests.TransactionResult; import org.apache.kafka.common.requests.TxnOffsetCommitRequest; import org.apache.kafka.common.requests.TxnOffsetCommitResponse; -import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +60,8 @@ public class TransactionState { private static final Logger log = LoggerFactory.getLogger(TransactionState.class); + private static final int NO_INFLIGHT_REQUEST_CORRELATION_ID = -1; + private volatile PidAndEpoch pidAndEpoch; private final Map sequenceNumbers; private final String transactionalId; @@ -70,7 +71,7 @@ public class TransactionState { private final Set pendingPartitionsToBeAddedToTransaction; private final Set partitionsInTransaction; private final Map pendingTxnOffsetCommits; - private int inFlightRequestCorrelationId = Integer.MIN_VALUE; + private int inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID; private Node transactionCoordinator; private Node consumerGroupCoordinator; @@ -79,7 +80,7 @@ public class TransactionState { private volatile boolean isInitializing = false; private volatile boolean isFenced = false; - public TransactionState(Time time, String transactionalId, int transactionTimeoutMs) { + public TransactionState(String transactionalId, int transactionTimeoutMs) { pidAndEpoch = new PidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); sequenceNumbers = new HashMap<>(); this.transactionalId = transactionalId; @@ -98,6 +99,10 @@ public int compare(TransactionalRequest o1, TransactionalRequest o2) { this.pendingTxnOffsetCommits = new HashMap<>(); } + public TransactionState() { + this("", 0); + } + public static class TransactionalRequest { private enum Priority { FIND_COORDINATOR(0), @@ -201,12 +206,9 @@ public TransactionalRequest nextTransactionalRequest() { return pendingTransactionalRequests.poll(); } - public TransactionState(Time time) { - this(time, "", 0); - } public boolean isTransactional() { - return !transactionalId.isEmpty(); + return transactionalId != null && !transactionalId.isEmpty(); } public String transactionalId() { @@ -287,7 +289,7 @@ public Node coordinator(FindCoordinatorRequest.CoordinatorType type) { case TRANSACTION: return transactionCoordinator; default: - return null; + throw new IllegalStateException("Received an invalid coordinator type: " + type); } } @@ -302,6 +304,9 @@ private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, Strin break; case TRANSACTION: transactionCoordinator = null; + break; + default: + throw new IllegalStateException("Got an invalid coordintor type: " + type); } pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, false)); } @@ -312,11 +317,11 @@ public void setInFlightRequestCorrelationId(int correlationId) { } public void resetInFlightRequestCorrelationId() { - inFlightRequestCorrelationId = Integer.MIN_VALUE; + inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID; } public boolean hasInflightTransactionalRequest() { - return inFlightRequestCorrelationId != Integer.MIN_VALUE; + return inFlightRequestCorrelationId != NO_INFLIGHT_REQUEST_CORRELATION_ID; } // visible for testing @@ -366,7 +371,7 @@ public synchronized void setPidAndEpoch(long pid, short epoch) { } /** - * This method is used when the producer needs to reset it's internal state because of an irrecoverable exception + * This method is used when the producer needs to reset its internal state because of an irrecoverable exception * from the broker. * * We need to reset the producer id and associated state when we have sent a batch to the broker, but we either get @@ -376,17 +381,15 @@ public synchronized void setPidAndEpoch(long pid, short epoch) { * In all of these cases, we don't know whether batch was actually committed on the broker, and hence whether the * sequence number was actually updated. If we don't reset the producer state, we risk the chance that all future * messages will return an OutOfOrderSequenceException. + * + * Note that we can't reset the producer state for the transactional producer as this would mean bumping the epoch + * for the same pid. This might involve aborting the ongoing transaction during the initPidRequest, and the user + * would not have any way of knowing this happened. So for the transactional producer, it's best to return the + * produce error to the user and let them abort the transaction and close the producer explicitly. */ public synchronized void resetProducerId() { - if (isTransactional()) { - // We can't reset the producer state for the transactional producer as this would mean bumping the epoch - // for the same pid. This might involve aborting the ongoing transaction during the initPidRequest, and - // the user would not have any way of knowing this happened. - // - // So it's best to return the produce error to the user and let them abort the transaction and close - // the producer explicitly. + if (isTransactional()) return; - } setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); this.sequenceNumbers.clear(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index efc983faea246..72fcd9fc1345a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -546,14 +546,11 @@ public void awaitFlushCompletion() throws InterruptedException { } public boolean hasUnflushedBatches() { - boolean hasUnflushed = false; for (Map.Entry> entry : this.batches().entrySet()) { - if (0 < entry.getValue().size()) { - hasUnflushed = true; - break; - } + if (!entry.getValue().isEmpty()) + return true; } - return 0 < this.incomplete.incomplete.size() || hasUnflushed; + return !this.incomplete.incomplete.isEmpty(); } /** diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 935f0e6b6c60c..c8682cd5d2839 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -246,8 +246,6 @@ public void close() { if (builtRecords != null) return; - closeForRecordAppends(); - if (isTransactional && producerId == RecordBatch.NO_PRODUCER_ID) throw new IllegalArgumentException("Cannot write transactional messages without a valid producer ID"); @@ -262,6 +260,8 @@ public void close() { throw new IllegalArgumentException("Idempotent messages are not supported for magic " + magic); } + closeForRecordAppends(); + if (numRecords == 0L) { buffer().position(initPos); builtRecords = MemoryRecords.EMPTY; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index 8265e04690500..482811ed9174f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -55,7 +55,7 @@ public static class Builder extends AbstractRequest.Builder { private final short acks; private final int timeout; private final Map partitionRecords; - private final String transactioanlId; + private final String transactionalId; public Builder(byte magic, short acks, @@ -67,7 +67,7 @@ public Builder(byte magic, this.acks = acks; this.timeout = timeout; this.partitionRecords = partitionRecords; - this.transactioanlId = transactionalId; + this.transactionalId = transactionalId; } public Builder(byte magic, @@ -82,7 +82,7 @@ public ProduceRequest build(short version) { if (version < 2) throw new UnsupportedVersionException("ProduceRequest versions older than 2 are not supported."); - return new ProduceRequest(version, acks, timeout, partitionRecords, transactioanlId); + return new ProduceRequest(version, acks, timeout, partitionRecords, transactionalId); } @Override @@ -93,8 +93,8 @@ public String toString() { .append(", acks=").append(acks) .append(", timeout=").append(timeout) .append(", partitionRecords=(").append(partitionRecords) - .append(", transactionalId=(").append(transactioanlId != null ? transactioanlId : "") - .append("))"); + .append("), transactionalId='").append(transactionalId != null ? transactionalId : "") + .append("'"); return bld.toString(); } } @@ -110,10 +110,6 @@ public String toString() { // Care should be taken in methods that use this field. private volatile Map partitionRecords; - private ProduceRequest(short version, short acks, int timeout, Map partitionRecords) { - this(version, acks, timeout, partitionRecords, null); - } - private ProduceRequest(short version, short acks, int timeout, Map partitionRecords, String transactionalId) { super(version); this.acks = acks; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index 1ee1953a04fff..8e4935016eaaa 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -1565,8 +1565,8 @@ private KafkaConsumer newConsumer(Time time, subscriptions, metadata, retryBackoffMs, - requestTimeoutMs, - "READ_UNCOMMITTED"); + requestTimeoutMs + ); } private static class FetchInfo { diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java index d152117034dfe..7d443b9b70b72 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java @@ -549,7 +549,7 @@ public void testIdempotenceWithOldMagic() throws InterruptedException { apiVersions.update("foobar", NodeApiVersions.create(Arrays.asList(new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE.id, (short) 0, (short) 2)))); RecordAccumulator accum = new RecordAccumulator(batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * batchSize, - CompressionType.NONE, 10, 100L, metrics, time, apiVersions, new TransactionState(time)); + CompressionType.NONE, 10, 100L, metrics, time, apiVersions, new TransactionState()); accum.append(tp1, 0L, key, value, null, 0); } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 0d19aa05ca1fe..f0419914d7019 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -377,7 +377,7 @@ public void testMetadataTopicExpiry() throws Exception { @Test public void testInitPidRequest() throws Exception { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(new MockTime()); + TransactionState transactionState = new TransactionState(); setupWithTransactionState(transactionState); client.setNode(new Node(1, "localhost", 33343)); client.prepareResponse(new MockClient.RequestMatcher() { @@ -395,7 +395,7 @@ public boolean matches(AbstractRequest body) { @Test public void testSequenceNumberIncrement() throws InterruptedException { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(new MockTime()); + TransactionState transactionState = new TransactionState(); transactionState.setPidAndEpoch(producerId, (short) 0); setupWithTransactionState(transactionState); client.setNode(new Node(1, "localhost", 33343)); @@ -448,7 +448,7 @@ public boolean matches(AbstractRequest body) { @Test public void testAbortRetryWhenPidChanges() throws InterruptedException { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(new MockTime()); + TransactionState transactionState = new TransactionState(); transactionState.setPidAndEpoch(producerId, (short) 0); setupWithTransactionState(transactionState); client.setNode(new Node(1, "localhost", 33343)); @@ -497,7 +497,7 @@ public void testAbortRetryWhenPidChanges() throws InterruptedException { @Test public void testResetWhenOutOfOrderSequenceReceived() throws InterruptedException { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(new MockTime()); + TransactionState transactionState = new TransactionState(); transactionState.setPidAndEpoch(producerId, (short) 0); setupWithTransactionState(transactionState); client.setNode(new Node(1, "localhost", 33343)); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java index a8a17167d99af..2b0e7275e6e7b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java @@ -19,7 +19,6 @@ import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.utils.MockTime; import org.junit.Before; import org.junit.Test; @@ -36,13 +35,13 @@ public void setUp() { @Test(expected = IllegalStateException.class) public void testInvalidSequenceIncrement() { - TransactionState transactionState = new TransactionState(new MockTime()); + TransactionState transactionState = new TransactionState(); transactionState.incrementSequenceNumber(topicPartition, 3333); } @Test public void testDefaultSequenceNumber() { - TransactionState transactionState = new TransactionState(new MockTime()); + TransactionState transactionState = new TransactionState(); assertEquals((int) transactionState.sequenceNumber(topicPartition), 0); transactionState.incrementSequenceNumber(topicPartition, 3); assertEquals((int) transactionState.sequenceNumber(topicPartition), 3); @@ -51,7 +50,7 @@ public void testDefaultSequenceNumber() { @Test public void testProducerIdReset() { - TransactionState transactionState = new TransactionState(new MockTime()); + TransactionState transactionState = new TransactionState(); assertEquals((int) transactionState.sequenceNumber(topicPartition), 0); transactionState.incrementSequenceNumber(topicPartition, 3); assertEquals((int) transactionState.sequenceNumber(topicPartition), 3); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java index 0a88b8df3d11e..2d665aefc37cc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java @@ -98,7 +98,7 @@ public void setup() { int batchSize = 16 * 1024; MetricConfig metricConfig = new MetricConfig().tags(metricTags); this.brokerNode = new Node(0, "localhost", 2211); - this.transactionState = new TransactionState(time, transactionalId, transactionTimeoutMs); + this.transactionState = new TransactionState(transactionalId, transactionTimeoutMs); Metrics metrics = new Metrics(metricConfig, time); this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionState); this.sender = new Sender(this.client, diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala index 99aca99c833e3..73e48af9434c8 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala @@ -156,7 +156,8 @@ class FetchRequestTest extends BaseRequestTest { val (topicPartition, leaderId) = createTopics(numTopics = 1, numPartitions = 1).head producer.send(new ProducerRecord(topicPartition.topic, topicPartition.partition, "key", new String(new Array[Byte](maxPartitionBytes + 1)))).get - val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(maxPartitionBytes, Seq(topicPartition))).build(2) + val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(maxPartitionBytes, + Seq(topicPartition))).build(2) val fetchResponse = sendFetchRequest(leaderId, fetchRequest, version = 2) val partitionData = fetchResponse.responseData.get(topicPartition) assertEquals(Errors.NONE, partitionData.error) From 866684f37460889aec75ce5ce4ee119f58825413 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 13 Apr 2017 17:39:00 -0700 Subject: [PATCH 06/21] Fix FindBugs errors --- .../org/apache/kafka/clients/producer/TransactionState.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index 964c3e6c3bd4a..fbbc24e2dd4cb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -530,8 +530,6 @@ public void handleResponse(AbstractResponse responseBody) { reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); - } else if (error == Errors.INVALID_TRANSACTION_TIMEOUT) { - result.setError(error.exception()); } else { result.setError(error.exception()); } From d241dc0f665f4f0e9e96673adfef33dfc843410a Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 18 Apr 2017 11:12:59 -0700 Subject: [PATCH 07/21] Fix correctness issues in Fetcher and KafkaProducer which were pointed out in the PR --- .../apache/kafka/clients/consumer/internals/Fetcher.java | 8 ++++---- .../org/apache/kafka/clients/producer/KafkaProducer.java | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 4327337c7a4e2..f3f71ec2a5678 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -1085,11 +1085,11 @@ public int compare(FetchResponse.AbortedTransaction o1, FetchResponse.AbortedTra private boolean containsAbortMarker(RecordBatch batch) { Iterator batchIterator = batch.iterator(); Record firstRecord = batchIterator.hasNext() ? batchIterator.next() : null; - if (firstRecord != null && batchIterator.hasNext()) - throw new CorruptRecordException("A RecordBatch containing a control message contained more than one message."); - return firstRecord != null && firstRecord.isControlRecord() && ControlRecordType.ABORT == ControlRecordType.parse(firstRecord.key()); + boolean containsAbortMarker = firstRecord != null && firstRecord.isControlRecord() && ControlRecordType.ABORT == ControlRecordType.parse(firstRecord.key()); + if (containsAbortMarker && batchIterator.hasNext()) + throw new CorruptRecordException("A record batch containing a control message contained more than one record."); + return containsAbortMarker; } - } private static class ExceptionMetadata { diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 5669a47994e01..26b1675ca1d6f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -637,9 +637,6 @@ private Future doSend(ProducerRecord record, Callback call throw new IllegalStateException("Cannot call send while a commit or abort is in progress."); } - if (transactionState != null && transactionState.isInTransaction()) { - transactionState.maybeAddPartitionToTransaction(new TopicPartition(record.topic(), record.partition())); - } TopicPartition tp = null; try { @@ -673,6 +670,12 @@ private Future doSend(ProducerRecord record, Callback call log.trace("Sending record {} with callback {} to topic {} partition {}", record, callback, record.topic(), partition); // producer callback will make sure to call both 'callback' and interceptor callback Callback interceptCallback = this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + + if (transactionState != null && transactionState.isInTransaction()) { + transactionState.maybeAddPartitionToTransaction(new TopicPartition(record.topic(), record.partition())); + } + + RecordAccumulator.RecordAppendResult result = accumulator.append(tp, timestamp, serializedKey, serializedValue, interceptCallback, remainingWaitMs); if (result.batchIsFull || result.newBatchCreated) { From a9ab31843707d4041cfbc0ca5bfa2468a7fd366e Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 18 Apr 2017 11:44:28 -0700 Subject: [PATCH 08/21] Address a few more PR comments. Rebase on trunk / fix build and test failures --- .../apache/kafka/clients/consumer/internals/Fetcher.java | 4 +++- core/src/test/scala/unit/kafka/log/LogTest.scala | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index f3f71ec2a5678..9768c6f66ef5b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -996,11 +996,13 @@ private Record nextFetchedRecord() { return null; } currentBatch = batches.next(); + + maybeEnsureValid(currentBatch); + if (isBatchAborted(currentBatch, abortedPids, abortedTransactions)) { continue; } - maybeEnsureValid(currentBatch); records = currentBatch.streamingIterator(); } diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index 0b1d2992863b9..b5e376e0235e8 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -434,22 +434,22 @@ class LogTest { val buffer = ByteBuffer.allocate(512) - var builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 0L, time.milliseconds(), 1L, epoch, 0, 0) + var builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 0L, time.milliseconds(), 1L, epoch, 0, false, 0) builder.append(new SimpleRecord("key".getBytes, "value".getBytes)) builder.close() // Append a record with other pids. - builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 1L, time.milliseconds(), 2L, epoch, 0, 0) + builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 1L, time.milliseconds(), 2L, epoch, 0, false, 0) builder.append(new SimpleRecord("key".getBytes, "value".getBytes)) builder.close() // Append a record with other pids. - builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 2L, time.milliseconds(), 3L, epoch, 0, 0) + builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 2L, time.milliseconds(), 3L, epoch, 0, false, 0) builder.append(new SimpleRecord("key".getBytes, "value".getBytes)) builder.close() // Append a record with other pids. - builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 3L, time.milliseconds(), 4L, epoch, 0, 0) + builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 3L, time.milliseconds(), 4L, epoch, 0, false, 0) builder.append(new SimpleRecord("key".getBytes, "value".getBytes)) builder.close() From caac46fbf6aa8b59368c345086218bc42218baea Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 20 Apr 2017 17:37:33 -0700 Subject: [PATCH 09/21] WIP --- .../org/apache/kafka/clients/producer/TransactionState.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index fbbc24e2dd4cb..8b5befd009add 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -339,6 +339,8 @@ public synchronized FutureTransactionalResult initializeTransactions() { throw new IllegalStateException("Multiple concurrent calls to initTransactions are not allowed."); } isInitializing = true; + setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); + this.sequenceNumbers.clear(); if (transactionCoordinator == null) pendingTransactionalRequests.add(findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId, false)); From 4c4380e4b3c474934043da02197cafe21d5c0253 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 25 Apr 2017 18:17:18 -0700 Subject: [PATCH 10/21] Addressed PR comments. Main changes: 1. Introduced more formal states to TransactionState and made the state transition checks more formal. 2. Stopped throwing exceptions from completion handlers. 3. If the Transaction is in ERROR state, the next transactional method will fail. 4. The transaction can be in error state if any transactional request or produce request within the transaction had a fatal error. --- .../clients/consumer/ConsumerConfig.java | 7 +- .../clients/consumer/internals/Fetcher.java | 41 ++-- .../kafka/clients/producer/KafkaProducer.java | 95 ++++---- .../kafka/clients/producer/Producer.java | 2 +- .../clients/producer/TransactionState.java | 230 +++++++++++++----- .../clients/producer/internals/Sender.java | 19 +- .../apache/kafka/common/config/ConfigDef.java | 11 + .../producer/internals/TransactionsTest.java | 70 ++++++ 8 files changed, 331 insertions(+), 144 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index cba5a19f21aec..8643d07454f69 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -240,9 +240,10 @@ public class ConsumerConfig extends AbstractConfig { public static final String ISOLATION_LEVEL_DOC = "

Controls how to read messages written transactionally. If set to READ_COMMITTED, consumer.poll() will only return" + " transactional messages which have been committed. If set to READ_UNCOMMITTED' (the default), consumer.poll() will return all messages, even transactional messages" + " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.

Messages will always be returned in offset order. Hence, in " + - " READ_COMMITTED mode, consumer.poll() will only return messages upto the last resolved (committed or aborted) transaction. In particular any messages appearing after" + - " messages belonging to onging transactions will be withheld until the relevant transaction has been completed. As a result, READ_COMMITTED" + - " consumers will not be able to read up to the log end offset when there are inflight transactions.

"; + " READ_COMMITTED mode, consumer.poll() will only return messages up to the last stable offset (LSO), which is the one less than the offset of the first open transaction." + + " In particular any messages appearing after messages belonging to ongoing transactions will be withheld until the relevant transaction has been completed. As a result, READ_COMMITTED" + + " consumers will not be able to read up to the high watermark when there are in flight transactions.

Further, when in READ_COMMITTED the seekToEnd method will" + + " return the LSO"; public static final String DEFAULT_ISOLATION_LEVEL = IsolationLevel.READ_UNCOMMITTED.toString(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 9768c6f66ef5b..58e07be6b0ba7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -77,7 +77,6 @@ import java.util.List; import java.util.Map; import java.util.PriorityQueue; -import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; @@ -922,8 +921,8 @@ private class PartitionRecords { private final TopicPartition partition; private final CompletedFetch completedFetch; private final Iterator batches; - private final Set abortedPids; - private final Queue abortedTransactions; + private final Set abortedProducerIds; + private final PriorityQueue abortedTransactions; private int recordsRead; private int bytesRead; @@ -939,7 +938,7 @@ private PartitionRecords(TopicPartition partition, this.completedFetch = completedFetch; this.batches = batches; this.nextFetchOffset = completedFetch.fetchedOffset; - this.abortedPids = new HashSet<>(); + this.abortedProducerIds = new HashSet<>(); this.abortedTransactions = abortedTransactions(completedFetch.partitionData); } @@ -999,7 +998,7 @@ private Record nextFetchedRecord() { maybeEnsureValid(currentBatch); - if (isBatchAborted(currentBatch, abortedPids, abortedTransactions)) { + if (isolationLevel == IsolationLevel.READ_COMMITTED && isBatchAborted(currentBatch)) { continue; } @@ -1037,9 +1036,9 @@ private List> fetchRecords(int maxRecords) { return records; } - private boolean isBatchAborted(RecordBatch batch, Set abortedPids, Queue abortedTransactions) { + private boolean isBatchAborted(RecordBatch batch) { /* When in READ_COMMITTED mode, we need to do the following for each incoming entry: - * 0. Check whether the pid is in the 'abortedPids' set && the entry does not include an abort marker. + * 0. Check whether the pid is in the 'abortedProducerIds' set && the entry does not include an abort marker. * If so, skip the entry. * 1. If the pid is in aborted pids and the entry contains an abort marker, remove the pid from * aborted pids and skip the entry. @@ -1048,32 +1047,30 @@ private boolean isBatchAborted(RecordBatch batch, Set abortedPids, Queue abortedTransactions(FetchResponse.PartitionData partition) { + private PriorityQueue abortedTransactions(FetchResponse.PartitionData partition) { PriorityQueue abortedTransactions = null; - if (partition.abortedTransactions != null && 0 < partition.abortedTransactions.size()) { + if (partition.abortedTransactions != null && !partition.abortedTransactions.isEmpty()) { abortedTransactions = new PriorityQueue<>( partition.abortedTransactions.size(), new Comparator() { @Override public int compare(FetchResponse.AbortedTransaction o1, FetchResponse.AbortedTransaction o2) { - return (int) o1.firstOffset - (int) o2.firstOffset; + return Long.compare(o1.firstOffset, o2.firstOffset); } } ); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 26b1675ca1d6f..d61246af547f7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -21,7 +21,6 @@ import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.producer.internals.FutureTransactionalResult; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.Sender; @@ -47,7 +46,6 @@ import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.network.ChannelBuilder; import org.apache.kafka.common.network.Selector; -import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.AbstractRecords; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.RecordBatch; @@ -267,7 +265,7 @@ private KafkaProducer(ProducerConfig config, Serializer keySerializer, Serial this.maxBlockTimeMs = configureMaxBlockTime(config, userProvidedConfigs); this.requestTimeoutMs = configureRequestTimeout(config, userProvidedConfigs); - this.transactionState = configureTransactionState(config, time); + this.transactionState = configureTransactionState(config); int retries = configureRetries(config, transactionState != null); int maxInflightRequests = configureInflightRequests(config, transactionState != null); short acks = configureAcks(config, transactionState != null); @@ -367,9 +365,26 @@ private static int configureRequestTimeout(ProducerConfig config, Map offsets, String consumerGroupId) throws ProducerFencedException { - if (transactionState == null || !transactionState.isInTransaction() || transactionState.isCompletingTransaction() - || !transactionState.hasPid()) - throw new IllegalStateException("Cannot send offsets to transaction. Either transactions are not enabled, or a transaction" + - " isn't active right now, or a transaction is in the process of being completed, or the producer" + - " has been fenced."); - FutureTransactionalResult result = transactionState.sendOffsetsToTransaction(offsets, consumerGroupId); - result.get(); + if (transactionState == null) + throw new IllegalStateException("Cannot send offsets to transaction since transactions are not enabled."); + transactionState.sendOffsetsToTransaction(offsets, consumerGroupId).get(); } /** @@ -509,13 +515,9 @@ public void sendOffsetsToTransaction(Map offs * transactional.id is active. */ public void commitTransaction() throws ProducerFencedException { - if (transactionState == null || !transactionState.isInTransaction() || !transactionState.hasPid() - || transactionState.isCompletingTransaction()) - throw new IllegalStateException("Cannot commit transaction. Either transactions are not enabled, or there is" + - " no transaction in progress, or the producer is fenced, or a transaction is already completing."); - FutureTransactionalResult result = transactionState.beginCommittingTransaction(); - result.get(); - + if (transactionState == null) + throw new IllegalStateException("Cannot commit transaction since transactions are not enabled"); + transactionState.beginCommittingTransaction().get(); } /** @@ -525,12 +527,9 @@ public void commitTransaction() throws ProducerFencedException { * transactional.id is active. */ public void abortTransaction() throws ProducerFencedException { - if (transactionState == null || !transactionState.isInTransaction() || !transactionState.hasPid() - || transactionState.isCompletingTransaction()) - throw new IllegalStateException("Cannot commit transaction. Either transactions are not enabled, or there is" + - " no transaction in progress, or the producer is fenced, or a transaction is already completing."); - FutureTransactionalResult result = transactionState.beginAbortingTransaction(); - result.get(); + if (transactionState == null) + throw new IllegalStateException("Cannot abort transaction since transactions are not enabled."); + transactionState.beginAbortingTransaction().get(); } /** @@ -626,17 +625,16 @@ public Future send(ProducerRecord record, Callback callbac * Implementation of asynchronously send a record to a topic. */ private Future doSend(ProducerRecord record, Callback callback) { - if (transactionState != null && transactionState.isTransactional() && !transactionState.hasPid()) { - if (transactionState.isFenced()) { - throw Errors.INVALID_PRODUCER_EPOCH.exception(); - } - throw new IllegalStateException("Cannot perform a 'send' before completing a call to initTransactions when transactions are enabled."); - } + if (transactionState != null) { + if (transactionState.isTransactional() && !transactionState.hasPid()) + throw new IllegalStateException("Cannot perform a 'send' before completing a call to initTransactions when transactions are enabled."); - if (transactionState != null && transactionState.isCompletingTransaction()) { - throw new IllegalStateException("Cannot call send while a commit or abort is in progress."); - } + if (transactionState.isFenced() && transactionState.isInTransaction()) + throw new ProducerFencedException("The current producer has been fenced off by a another producer using the same transactional id."); + if (transactionState.isCompletingTransaction()) + throw new IllegalStateException("Cannot call send while a commit or abort is in progress."); + } TopicPartition tp = null; try { @@ -669,7 +667,7 @@ private Future doSend(ProducerRecord record, Callback call long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); log.trace("Sending record {} with callback {} to topic {} partition {}", record, callback, record.topic(), partition); // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp); + Callback interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp, transactionState); if (transactionState != null && transactionState.isInTransaction()) { transactionState.maybeAddPartitionToTransaction(new TopicPartition(record.topic(), record.partition())); @@ -1011,12 +1009,14 @@ private static class InterceptorCallback implements Callback { private final Callback userCallback; private final ProducerInterceptors interceptors; private final TopicPartition tp; + private final TransactionState transactionState; public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp) { + TopicPartition tp, TransactionState transactionState) { this.userCallback = userCallback; this.interceptors = interceptors; this.tp = tp; + this.transactionState = transactionState; } public void onCompletion(RecordMetadata metadata, Exception exception) { @@ -1030,6 +1030,9 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { } if (this.userCallback != null) this.userCallback.onCompletion(metadata, exception); + + if (exception != null && transactionState != null) + transactionState.maybeSetError(exception); } } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java index 0554c9232d5b1..a057de9f50faf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java @@ -51,7 +51,7 @@ public interface Producer extends Closeable { * @throws IllegalStateException if the TransactionalId for the producer is not set * in the configuration. */ - void initTransactions() throws IllegalStateException; + void initTransactions(); /** * Should be called before the start of each new transaction. diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index 8b5befd009add..114f1e7326d35 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -23,6 +23,7 @@ import org.apache.kafka.clients.producer.internals.TransactionalRequestResult; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.ProducerFencedException; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; @@ -51,6 +52,13 @@ import java.util.PriorityQueue; import java.util.Set; +import static org.apache.kafka.clients.producer.TransactionState.State.ABORTING_TRANSACTION; +import static org.apache.kafka.clients.producer.TransactionState.State.COMMITTING_TRANSACTION; +import static org.apache.kafka.clients.producer.TransactionState.State.ERROR; +import static org.apache.kafka.clients.producer.TransactionState.State.FENCED; +import static org.apache.kafka.clients.producer.TransactionState.State.INITIALIZING; +import static org.apache.kafka.clients.producer.TransactionState.State.IN_TRANSACTION; +import static org.apache.kafka.clients.producer.TransactionState.State.READY; import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_EPOCH; import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_ID; @@ -75,20 +83,48 @@ public class TransactionState { private Node transactionCoordinator; private Node consumerGroupCoordinator; - private volatile boolean isInTransaction = false; - private volatile boolean isCompletingTransaction = false; - private volatile boolean isInitializing = false; - private volatile boolean isFenced = false; + private State currentState = State.UNINITIALIZED; + + public enum State { + UNINITIALIZED, + INITIALIZING, + READY, + IN_TRANSACTION, + COMMITTING_TRANSACTION, + ABORTING_TRANSACTION, + FENCED, + ERROR; + + private boolean isTransitionValid(State source, State target) { + switch (target) { + case INITIALIZING: + return source == UNINITIALIZED || source == READY || source == ERROR; + case READY: + return source == INITIALIZING || source == COMMITTING_TRANSACTION || source == ABORTING_TRANSACTION; + case IN_TRANSACTION: + return source == READY; + case COMMITTING_TRANSACTION: + return source == IN_TRANSACTION; + case ABORTING_TRANSACTION: + return source == IN_TRANSACTION || source == ERROR; + default: + // We can transition to FENCED or ERROR unconditionally. + // FENCED is never a valid starting state for any transition. So the only option is to close the + // producer or do purely non transactional requests. + return true; + } + } + } public TransactionState(String transactionalId, int transactionTimeoutMs) { pidAndEpoch = new PidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); sequenceNumbers = new HashMap<>(); this.transactionalId = transactionalId; this.transactionTimeoutMs = transactionTimeoutMs; - this.pendingTransactionalRequests = new PriorityQueue<>(2, new Comparator() { + this.pendingTransactionalRequests = new PriorityQueue<>(10, new Comparator() { @Override public int compare(TransactionalRequest o1, TransactionalRequest o2) { - return o1.priority().priority() - o2.priority.priority(); + return Integer.compare(o1.priority().priority(), o2.priority.priority()); } }); this.transactionCoordinator = null; @@ -130,17 +166,19 @@ public int priority() { // a pending FindCoordinator request, that must always go first. Next, If we need a Pid, that must go second. // The endTxn request must always go last. private final Priority priority; + private final TransactionalRequestResult result; private boolean isRetry; private TransactionalRequest(AbstractRequest.Builder requestBuilder, RequestCompletionHandler handler, FindCoordinatorRequest.CoordinatorType coordinatorType, Priority priority, - boolean isRetry, String coordinatorKey) { + boolean isRetry, String coordinatorKey, TransactionalRequestResult result) { this.requestBuilder = requestBuilder; this.handler = handler; this.coordinatorType = coordinatorType; this.priority = priority; this.isRetry = isRetry; this.coordinatorKey = coordinatorKey; + this.result = result; } public AbstractRequest.Builder requestBuilder() { @@ -167,6 +205,15 @@ public boolean isEndTxnRequest() { return priority == Priority.END_TXN; } + public boolean maybeTerminateWithError(RuntimeException error) { + if (result != null) { + result.setError(error); + result.done(); + return true; + } + return false; + } + private void setRetry() { isRetry = true; } @@ -195,7 +242,6 @@ public boolean hasPendingTransactionalRequests() { && newPartitionsToBeAddedToTransaction.isEmpty()); } - public TransactionalRequest nextTransactionalRequest() { if (!hasPendingTransactionalRequests()) return null; @@ -206,6 +252,25 @@ public TransactionalRequest nextTransactionalRequest() { return pendingTransactionalRequests.poll(); } + public boolean isInErrorState() { + return currentState == ERROR; + } + + + private boolean transitionTo(State target) { + if (currentState.isTransitionValid(currentState, target)) { + currentState = target; + return true; + } else { + log.error("Invalid transition attempted from state {} to state {}.", currentState.name(), target.name()); + } + return false; + } + + private void ensureTransactional() { + if (!isTransactional()) + throw new IllegalStateException("Transactional method invoked on a non-transactional producer."); + } public boolean isTransactional() { return transactionalId != null && !transactionalId.isEmpty(); @@ -216,46 +281,56 @@ public String transactionalId() { } public boolean hasPid() { - return pidAndEpoch.isValid() && !isFenced; + return pidAndEpoch.isValid(); } public boolean isFenced() { - return isFenced; + return currentState == FENCED; } public void beginTransaction() { - isInTransaction = true; + ensureTransactional(); + if (!transitionTo(IN_TRANSACTION)) + throw new IllegalStateException("Producer isn't ready to begin a transaction."); } public boolean isCompletingTransaction() { - return isInTransaction && isCompletingTransaction; + return currentState == COMMITTING_TRANSACTION || currentState == ABORTING_TRANSACTION; } public synchronized FutureTransactionalResult beginCommittingTransaction() { + ensureTransactional(); + if (!transitionTo(COMMITTING_TRANSACTION)) + throw new IllegalStateException("Cannot commit transaction, either because a transaction is already " + + "being completed at the moment, or because there has been an error with a previous request."); return beginCompletingTransaction(true); } public synchronized FutureTransactionalResult beginAbortingTransaction() { + ensureTransactional(); + if (!transitionTo(ABORTING_TRANSACTION)) + throw new IllegalStateException("Cannot abort transaction, either because a transaction is already " + + "being completed at the moment, or because there has been an error with a previous request."); return beginCompletingTransaction(false); } private FutureTransactionalResult beginCompletingTransaction(boolean isCommit) { - if (!isCompletingTransaction) { - TransactionalRequestResult result = new TransactionalRequestResult(); - FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); + TransactionalRequestResult result = new TransactionalRequestResult(); + FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); - isCompletingTransaction = true; - if (!newPartitionsToBeAddedToTransaction.isEmpty()) { - pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false)); - } - pendingTransactionalRequests.add(endTxnRequest(isCommit, false, result)); - return resultFuture; + if (!newPartitionsToBeAddedToTransaction.isEmpty()) { + pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false)); } - return null; + pendingTransactionalRequests.add(endTxnRequest(isCommit, false, result)); + return resultFuture; } public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map offsets, - String consumerGroupId) { + String consumerGroupId) { + ensureTransactional(); + if (!isInTransaction()) + throw new IllegalStateException("Cannot send offsets to transaction either because the producer is not in an " + + "active transaction or because there has been an error with one or more previous requests."); TransactionalRequestResult result = new TransactionalRequestResult(); FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); pendingTransactionalRequests.add(addOffsetsToTxnRequest(offsets, consumerGroupId, false, result)); @@ -263,10 +338,9 @@ public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map(pendingPartitionsToBeAddedToTransaction)); return new TransactionalRequest(builder, new AddPartitionsToTransactionCallback(), - FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, transactionalId); + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, transactionalId, null); } private TransactionalRequest findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey, boolean isRetry) { FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder(type, coordinatorKey); return new TransactionalRequest(builder, new FindCoordinatorCallback(type, coordinatorKey), - null, TransactionalRequest.Priority.FIND_COORDINATOR, isRetry, null); + null, TransactionalRequest.Priority.FIND_COORDINATOR, isRetry, null, null); } private TransactionalRequest endTxnRequest(boolean isCommit, boolean isRetry, TransactionalRequestResult result) { EndTxnRequest.Builder builder = new EndTxnRequest.Builder(transactionalId, pidAndEpoch.producerId, pidAndEpoch.epoch, isCommit ? TransactionResult.COMMIT : TransactionResult.ABORT); return new TransactionalRequest(builder, new EndTxnCallback(isCommit, result), - FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.END_TXN, isRetry, transactionalId); + FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.END_TXN, isRetry, transactionalId, result); } private TransactionalRequest addOffsetsToTxnRequest(Map offsets, @@ -456,7 +544,7 @@ private TransactionalRequest addOffsetsToTxnRequest(Map offsets, @@ -473,7 +561,7 @@ private TransactionalRequest txnOffsetCommitRequest(String consumerGroupId, bool TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(consumerGroupId, pidAndEpoch.producerId, pidAndEpoch.epoch, OffsetCommitRequest.DEFAULT_RETENTION_TIME, pendingTxnOffsetCommits); return new TransactionalRequest(builder, new TxnOffsetCommitCallback(consumerGroupId, result), - FindCoordinatorRequest.CoordinatorType.GROUP, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, consumerGroupId); + FindCoordinatorRequest.CoordinatorType.GROUP, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, consumerGroupId, result); } private abstract class TransactionalRequestCallBack implements RequestCompletionHandler { @@ -485,8 +573,11 @@ private abstract class TransactionalRequestCallBack implements RequestCompletion @Override public void onComplete(ClientResponse response) { - if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) - throw new IllegalStateException("Cannot have more than one transactional request in flight."); + if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) { + log.error("Detected more than one inflight transactional request. This should never happen."); + transitionTo(ERROR); + } + resetInFlightRequestCorrelationId(); if (response.wasDisconnected()) { reenqueue(); @@ -494,18 +585,18 @@ public void onComplete(ClientResponse response) { if (result != null) { result.setError(Errors.UNSUPPORTED_VERSION.exception()); result.done(); - } else { - throw Errors.UNSUPPORTED_VERSION.exception(); } + log.error("Could not execute transactional request because the broker isn't on the right version."); + transitionTo(ERROR); } else if (response.hasResponse()) { handleResponse(response.responseBody()); } else { if (result != null) { result.setError(Errors.UNKNOWN.exception()); result.done(); - } else { - throw Errors.UNKNOWN.exception(); } + log.error("Could not execute transactional request for unknown reasons"); + transitionTo(ERROR); } } @@ -526,7 +617,7 @@ public void handleResponse(AbstractResponse responseBody) { Errors error = initPidResponse.error(); if (error == Errors.NONE) { setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch()); - isInitializing = false; + transitionTo(READY); } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); @@ -534,6 +625,7 @@ public void handleResponse(AbstractResponse responseBody) { reenqueue(); } else { result.setError(error.exception()); + transitionTo(ERROR); } if (error == Errors.NONE || !result.isSuccessful()) @@ -565,14 +657,18 @@ public void handleResponse(AbstractResponse response) { } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { - throw error.exception(); + log.error("Seems like the broker has bad transaction state. producerId: {}, error: {}. message: {}", + pidAndEpoch.producerId, error, error.message()); + transitionTo(ERROR); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - isFenced = true; - throw error.exception(); + transitionTo(FENCED); + log.error("Epoch has become invalid: producerId: {}. epoch: {}. Message: {}", pidAndEpoch.producerId, pidAndEpoch.epoch, error.message()); } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) { - throw error.exception(); + transitionTo(ERROR); + log.error("No permissions add some partitions to the transaction: {}", error.message()); } else { - throw Errors.UNKNOWN.exception(); + transitionTo(ERROR); + log.error("Could not add partitions to transaction due to unknown error: {}", error.message()); } } @@ -606,9 +702,13 @@ public void handleResponse(AbstractResponse responseBody) { } else if (findCoordinatorResponse.error() == Errors.COORDINATOR_NOT_AVAILABLE) { reenqueue(); } else if (findCoordinatorResponse.error() == Errors.GROUP_AUTHORIZATION_FAILED) { - throw Errors.GROUP_AUTHORIZATION_FAILED.exception(); + transitionTo(ERROR); + log.error("Not authorized to access the group with type {} and key {}. Message: {} ", type, + coordinatorKey, findCoordinatorResponse.error().message()); } else { - throw Errors.UNKNOWN.exception(); + transitionTo(ERROR); + log.error("Could not find a coordinator with type {} for unknown reasons. coordinatorKey: {}", type, + coordinatorKey, findCoordinatorResponse.error().message()); } } @@ -637,13 +737,12 @@ public void handleResponse(AbstractResponse responseBody) { reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); - } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { - result.setError(error.exception()); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - isFenced = true; + transitionTo(FENCED); result.setError(error.exception()); } else { result.setError(error.exception()); + transitionTo(ERROR); } if (error == Errors.NONE || !result.isSuccessful()) @@ -677,12 +776,11 @@ public void handleResponse(AbstractResponse responseBody) { reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); - } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { - result.setError(error.exception()); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - isFenced = true; + transitionTo(FENCED); result.setError(error.exception()); } else { + transitionTo(ERROR); result.setError(error.exception()); } @@ -721,9 +819,13 @@ public void handleResponse(AbstractResponse responseBody) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP, consumerGroupId); } } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - isFenced = true; + transitionTo(FENCED); result.setError(error.exception()); break; + } else { + result.setError(error.exception()); + transitionTo(ERROR); + break; } } @@ -734,9 +836,9 @@ public void handleResponse(AbstractResponse responseBody) { return; } - if (0 < pendingTxnOffsetCommits.size()) { + if (!pendingTxnOffsetCommits.isEmpty()) pendingTransactionalRequests.add(txnOffsetCommitRequest(consumerGroupId, true, result)); - } + } @Override diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 12d585c6e68b4..bda0fc0bb4bfd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -28,6 +28,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InvalidMetadataException; +import org.apache.kafka.common.errors.InvalidTxnStateException; import org.apache.kafka.common.errors.OutOfOrderSequenceException; import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TopicAuthorizationException; @@ -285,30 +286,33 @@ private boolean maybeSendTransactionalRequest(long now) { if (nextRequest.isEndTxnRequest() && transactionState.isCompletingTransaction() && accumulator.hasUnflushedBatches()) { if (!accumulator.flushInProgress()) accumulator.beginFlush(); - transactionState.didNotSend(nextRequest); + transactionState.reenqueue(nextRequest); + return false; + } + + if (nextRequest.isEndTxnRequest() && transactionState.isInErrorState()) { + nextRequest.maybeTerminateWithError(new InvalidTxnStateException("Cannot commit transaction when there are " + + "request errors. Please check your logs for the details of the errors encountered.")); return false; } Node targetNode = null; - long expiryTime = now + requestTimeout; - long nextIterationTime = now; - while (targetNode == null && nextIterationTime < expiryTime) { + while (targetNode == null) { try { - long timeRemaining = expiryTime - nextIterationTime; if (nextRequest.needsCoordinator()) { targetNode = transactionState.coordinator(nextRequest.coordinatorType()); if (targetNode == null) { transactionState.needsCoordinator(nextRequest); break; } - if (!NetworkClientUtils.awaitReady(client, targetNode, time, timeRemaining)) { + if (!NetworkClientUtils.awaitReady(client, targetNode, time, requestTimeout)) { transactionState.needsCoordinator(nextRequest); targetNode = null; break; } } else { - targetNode = awaitLeastLoadedNodeReady(timeRemaining); + targetNode = awaitLeastLoadedNodeReady(requestTimeout); } if (targetNode != null) { if (nextRequest.isRetry()) { @@ -325,7 +329,6 @@ private boolean maybeSendTransactionalRequest(long now) { } time.sleep(retryBackoffMs); metadata.requestUpdate(); - nextIterationTime = time.milliseconds(); } if (targetNode == null) diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java index aac5b53f1b71d..b3f6ce067b0a9 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java +++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java @@ -899,6 +899,17 @@ public String toString() { } } + public static class NonEmptyString implements Validator { + + @Override + public void ensureValid(String name, Object o) { + String s = (String) o; + if (s.isEmpty()) { + throw new ConfigException(name, o, "String must be non-empty"); + } + } + } + public static class ConfigKey { public final String name; public final Type type; diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java index 2d665aefc37cc..15f1c392b8b1f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java @@ -20,11 +20,13 @@ import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.MockClient; import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidTxnStateException; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -67,6 +69,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TransactionsTest { private static final int MAX_REQUEST_SIZE = 1024 * 1024; @@ -412,6 +415,73 @@ public void testProducerFencedException() throws InterruptedException, Execution responseFuture.get(); } + @Test + public void testDisallowCommitOnProduceFailure() throws InterruptedException { + client.setNode(brokerNode); + // This is called from the initTransactions method in the producer as the first order of business. + // It finds the coordinator and then gets a PID. + final long pid = 13131L; + final short epoch = 1; + transactionState.initializeTransactions(); + prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); + + sender.run(time.milliseconds()); // find coordinator + assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + + prepareInitPidResponse(Errors.NONE, false, pid, epoch); + + sender.run(time.milliseconds()); // get pid. + + assertTrue(transactionState.hasPid()); + transactionState.beginTransaction(); + transactionState.maybeAddPartitionToTransaction(tp0); + + Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), new MockCallback(transactionState), MAX_BLOCK_TIMEOUT).future; + + FutureTransactionalResult commitResult = transactionState.beginCommittingTransaction(); + assertFalse(responseFuture.isDone()); + prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); + prepareProduceResponse(Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, pid, epoch); + + sender.run(time.milliseconds()); // Send AddPartitionsRequest + assertFalse(commitResult.isDone()); + + sender.run(time.milliseconds()); // Send Produce Request, returns OutOfOrderSequenceException. + sender.run(time.milliseconds()); // try to commit. + assertTrue(commitResult.isDone()); // commit should be cancelled with exception without being sent. + + try { + commitResult.get(); + fail(); // the get() must throw an exception. + } catch (RuntimeException e) { + assertTrue(e instanceof InvalidTxnStateException); + } + + // Commit is not allowed, so let's abort and try again. + FutureTransactionalResult abortResult = transactionState.beginAbortingTransaction(); + prepareEndTxnResponse(Errors.NONE, TransactionResult.ABORT, pid, epoch); + sender.run(time.milliseconds()); // Send abort request. It is valid to transition from ERROR to ABORT + + assertTrue(abortResult.isDone()); + assertTrue(abortResult.get().isSuccessful()); + assertTrue(transactionState.isReadyForTransaction()); // make sure we are ready for a transaction now. + } + + private static class MockCallback implements Callback { + private final TransactionState transactionState; + public MockCallback(TransactionState transactionState) { + this.transactionState = transactionState; + } + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null && transactionState != null) { + transactionState.maybeSetError(exception); + } + + } + } + private void prepareFindCoordinatorResponse(Errors error, boolean shouldDisconnect, final FindCoordinatorRequest.CoordinatorType coordinatorType, final String coordinatorKey) { From 5142a1be541eabecfdba8eab69c0c26775e203df Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 25 Apr 2017 22:51:31 -0700 Subject: [PATCH 11/21] Cached the last error in the transaction state so that if we invoke another transactional method from an error state the cause can be included in the 'IllegalStateException'. Also moved around the methods in TransactionState so that they are more logically grouped. --- .../clients/producer/TransactionState.java | 383 ++++++++++-------- .../clients/producer/internals/Sender.java | 3 +- gradle/findbugs-exclude.xml | 9 + 3 files changed, 217 insertions(+), 178 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index 114f1e7326d35..60a0c1a341384 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -83,7 +83,8 @@ public class TransactionState { private Node transactionCoordinator; private Node consumerGroupCoordinator; - private State currentState = State.UNINITIALIZED; + private volatile State currentState = State.UNINITIALIZED; + private Exception lastError = null; public enum State { UNINITIALIZED, @@ -237,80 +238,58 @@ public boolean isValid() { } } - public boolean hasPendingTransactionalRequests() { - return !(pendingTransactionalRequests.isEmpty() - && newPartitionsToBeAddedToTransaction.isEmpty()); - } - - public TransactionalRequest nextTransactionalRequest() { - if (!hasPendingTransactionalRequests()) - return null; - - if (!newPartitionsToBeAddedToTransaction.isEmpty()) - pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false)); - - return pendingTransactionalRequests.poll(); - } - - public boolean isInErrorState() { - return currentState == ERROR; - } - - - private boolean transitionTo(State target) { - if (currentState.isTransitionValid(currentState, target)) { - currentState = target; - return true; - } else { - log.error("Invalid transition attempted from state {} to state {}.", currentState.name(), target.name()); - } - return false; - } - - private void ensureTransactional() { - if (!isTransactional()) - throw new IllegalStateException("Transactional method invoked on a non-transactional producer."); - } - - public boolean isTransactional() { - return transactionalId != null && !transactionalId.isEmpty(); - } + public synchronized FutureTransactionalResult initializeTransactions() { + ensureTransactional(); + if (!transitionTo(INITIALIZING)) + throw new IllegalStateException("Could not initialize transactions. Either transactions have already been " + + "initialized or are being initialized."); + setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); + this.sequenceNumbers.clear(); + if (transactionCoordinator == null) + pendingTransactionalRequests.add(findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId, false)); - public String transactionalId() { - return transactionalId; - } + TransactionalRequestResult result = new TransactionalRequestResult(); + FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); + if (!hasPid()) + pendingTransactionalRequests.add(initPidRequest(false, result)); + else + result.done(); - public boolean hasPid() { - return pidAndEpoch.isValid(); + return resultFuture; } - public boolean isFenced() { - return currentState == FENCED; - } - public void beginTransaction() { + public synchronized void beginTransaction() { ensureTransactional(); if (!transitionTo(IN_TRANSACTION)) throw new IllegalStateException("Producer isn't ready to begin a transaction."); } - public boolean isCompletingTransaction() { - return currentState == COMMITTING_TRANSACTION || currentState == ABORTING_TRANSACTION; - } - public synchronized FutureTransactionalResult beginCommittingTransaction() { ensureTransactional(); - if (!transitionTo(COMMITTING_TRANSACTION)) - throw new IllegalStateException("Cannot commit transaction, either because a transaction is already " + - "being completed at the moment, or because there has been an error with a previous request."); + if (!transitionTo(COMMITTING_TRANSACTION)) { + String msg = "Cannot commit transaction, either because a transaction is already " + + "being completed at the moment, or because there has been an error with a previous request."; + if (lastError != null) + throw new IllegalStateException(msg, lastError); + else + throw new IllegalStateException(msg); + } + return beginCompletingTransaction(true); } public synchronized FutureTransactionalResult beginAbortingTransaction() { ensureTransactional(); - if (!transitionTo(ABORTING_TRANSACTION)) - throw new IllegalStateException("Cannot abort transaction, either because a transaction is already " + - "being completed at the moment, or because there has been an error with a previous request."); + if (!transitionTo(ABORTING_TRANSACTION)) { + String msg = "Cannot abort transaction, either because a transaction is already " + + "being completed at the moment, or because there has been an error with a previous request."; + if (lastError != null) + throw new IllegalStateException(msg, lastError); + else + throw new IllegalStateException(msg); + } + return beginCompletingTransaction(false); } @@ -328,121 +307,27 @@ private FutureTransactionalResult beginCompletingTransaction(boolean isCommit) { public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map offsets, String consumerGroupId) { ensureTransactional(); - if (!isInTransaction()) - throw new IllegalStateException("Cannot send offsets to transaction either because the producer is not in an " + - "active transaction or because there has been an error with one or more previous requests."); + if (currentState != IN_TRANSACTION) { + String msg = "Cannot send offsets to transaction either because the producer is not in an " + + "active transaction or because there has been an error with one or more previous requests."; + if (lastError != null) + throw new IllegalStateException(msg, lastError); + else + throw new IllegalStateException(msg); + } + TransactionalRequestResult result = new TransactionalRequestResult(); FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); pendingTransactionalRequests.add(addOffsetsToTxnRequest(offsets, consumerGroupId, false, result)); return resultFuture; } - public boolean isInTransaction() { - return currentState == IN_TRANSACTION || isCompletingTransaction(); - } - public synchronized void maybeAddPartitionToTransaction(TopicPartition topicPartition) { if (partitionsInTransaction.contains(topicPartition)) return; newPartitionsToBeAddedToTransaction.add(topicPartition); } - public void needsRetry(TransactionalRequest request) { - request.setRetry(); - pendingTransactionalRequests.add(request); - } - - public void reenqueue(TransactionalRequest request) { - pendingTransactionalRequests.add(request); - } - - public Node coordinator(FindCoordinatorRequest.CoordinatorType type) { - switch (type) { - case GROUP: - return consumerGroupCoordinator; - case TRANSACTION: - return transactionCoordinator; - default: - throw new IllegalStateException("Received an invalid coordinator type: " + type); - } - } - - public void needsCoordinator(TransactionalRequest request) { - needsCoordinator(request.coordinatorType, request.coordinatorKey); - } - - private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) { - switch (type) { - case GROUP: - consumerGroupCoordinator = null; - break; - case TRANSACTION: - transactionCoordinator = null; - break; - default: - throw new IllegalStateException("Got an invalid coordinator type: " + type); - } - pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, false)); - } - - public boolean maybeSetError(Exception exception) { - if (isTransactional() && (isInTransaction() || isCompletingTransaction())) { - if (exception instanceof ProducerFencedException) - transitionTo(FENCED); - else - transitionTo(ERROR); - return true; - } - return false; - } - - public void setInFlightRequestCorrelationId(int correlationId) { - inFlightRequestCorrelationId = correlationId; - } - - public void resetInFlightRequestCorrelationId() { - inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID; - } - - public boolean hasInflightTransactionalRequest() { - return inFlightRequestCorrelationId != NO_INFLIGHT_REQUEST_CORRELATION_ID; - } - - // visible for testing - public boolean transactionContainsPartition(TopicPartition topicPartition) { - return isInTransaction() && partitionsInTransaction.contains(topicPartition); - } - - // visible for testing - public boolean hasPendingOffsetCommits() { - return isInTransaction() && !pendingTxnOffsetCommits.isEmpty(); - } - - // visible for testing - public boolean isReadyForTransaction() { - return isTransactional() && currentState == READY; - } - - public synchronized FutureTransactionalResult initializeTransactions() { - ensureTransactional(); - if (!transitionTo(INITIALIZING)) - throw new IllegalStateException("Could not initialize transactions. Either transactions have already been " + - "initialized or are being initialized."); - setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); - this.sequenceNumbers.clear(); - if (transactionCoordinator == null) - pendingTransactionalRequests.add(findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId, false)); - - TransactionalRequestResult result = new TransactionalRequestResult(); - FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); - if (!hasPid()) - pendingTransactionalRequests.add(initPidRequest(false, result)); - else - result.done(); - - return resultFuture; - } - /** * Get the current pid and epoch without blocking. Callers must use {@link PidAndEpoch#isValid()} to * verify that the result is valid. @@ -480,7 +365,8 @@ public synchronized void setPidAndEpoch(long pid, short epoch) { */ public synchronized void resetProducerId() { if (isTransactional()) - return; + throw new IllegalStateException("Cannot reset producer state for a transactional producer. " + + "You must either abort the ongoing transaction or reinitialize the transactional producer instead"); setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); this.sequenceNumbers.clear(); } @@ -506,8 +392,150 @@ public synchronized void incrementSequenceNumber(TopicPartition topicPartition, sequenceNumbers.put(topicPartition, currentSequenceNumber); } + public boolean hasPendingTransactionalRequests() { + return !(pendingTransactionalRequests.isEmpty() + && newPartitionsToBeAddedToTransaction.isEmpty()); + } + + public TransactionalRequest nextTransactionalRequest() { + if (!hasPendingTransactionalRequests()) + return null; + + if (!newPartitionsToBeAddedToTransaction.isEmpty()) + pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false)); + + return pendingTransactionalRequests.poll(); + } + + public String transactionalId() { + return transactionalId; + } + + public boolean hasPid() { + return pidAndEpoch.isValid(); + } + + public boolean isTransactional() { + return transactionalId != null && !transactionalId.isEmpty(); + } + + public boolean isFenced() { + return currentState == FENCED; + } + + public boolean isCompletingTransaction() { + return currentState == COMMITTING_TRANSACTION || currentState == ABORTING_TRANSACTION; + } + + public boolean isInTransaction() { + return currentState == IN_TRANSACTION || isCompletingTransaction(); + } + + public boolean isInErrorState() { + return currentState == ERROR; + } + + + public void needsRetry(TransactionalRequest request) { + request.setRetry(); + pendingTransactionalRequests.add(request); + } + + public void reenqueue(TransactionalRequest request) { + pendingTransactionalRequests.add(request); + } + + public Node coordinator(FindCoordinatorRequest.CoordinatorType type) { + switch (type) { + case GROUP: + return consumerGroupCoordinator; + case TRANSACTION: + return transactionCoordinator; + default: + throw new IllegalStateException("Received an invalid coordinator type: " + type); + } + } + + public void needsCoordinator(TransactionalRequest request) { + needsCoordinator(request.coordinatorType, request.coordinatorKey); + } + + public synchronized boolean maybeSetError(Exception exception) { + if (isTransactional() && isInTransaction()) { + if (exception instanceof ProducerFencedException) + transitionTo(FENCED, exception); + else + transitionTo(ERROR, exception); + return true; + } + return false; + } + + public void setInFlightRequestCorrelationId(int correlationId) { + inFlightRequestCorrelationId = correlationId; + } + + public void resetInFlightRequestCorrelationId() { + inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID; + } + + public boolean hasInflightTransactionalRequest() { + return inFlightRequestCorrelationId != NO_INFLIGHT_REQUEST_CORRELATION_ID; + } + + private boolean transitionTo(State target) { + return transitionTo(target, null); + } + + private boolean transitionTo(State target, Exception error) { + if (target == ERROR && error != null) + lastError = error; + if (currentState.isTransitionValid(currentState, target)) { + currentState = target; + return true; + } else { + log.error("Invalid transition attempted from state {} to state {}.", currentState.name(), target.name()); + } + return false; + } + + private void ensureTransactional() { + if (!isTransactional()) + throw new IllegalStateException("Transactional method invoked on a non-transactional producer."); + } + + private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) { + switch (type) { + case GROUP: + consumerGroupCoordinator = null; + break; + case TRANSACTION: + transactionCoordinator = null; + break; + default: + throw new IllegalStateException("Got an invalid coordinator type: " + type); + } + pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, false)); + } + + // visible for testing + public boolean transactionContainsPartition(TopicPartition topicPartition) { + return isInTransaction() && partitionsInTransaction.contains(topicPartition); + } + + // visible for testing + public boolean hasPendingOffsetCommits() { + return isInTransaction() && !pendingTxnOffsetCommits.isEmpty(); + } + + // visible for testing + public boolean isReadyForTransaction() { + return isTransactional() && currentState == READY; + } + private void completeTransaction() { transitionTo(READY); + lastError = null; partitionsInTransaction.clear(); } @@ -587,7 +615,7 @@ public void onComplete(ClientResponse response) { result.done(); } log.error("Could not execute transactional request because the broker isn't on the right version."); - transitionTo(ERROR); + transitionTo(ERROR, Errors.UNSUPPORTED_VERSION.exception()); } else if (response.hasResponse()) { handleResponse(response.responseBody()); } else { @@ -596,7 +624,7 @@ public void onComplete(ClientResponse response) { result.done(); } log.error("Could not execute transactional request for unknown reasons"); - transitionTo(ERROR); + transitionTo(ERROR, Errors.UNKNOWN.exception()); } } @@ -618,6 +646,7 @@ public void handleResponse(AbstractResponse responseBody) { if (error == Errors.NONE) { setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch()); transitionTo(READY); + lastError = null; } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); @@ -625,7 +654,7 @@ public void handleResponse(AbstractResponse responseBody) { reenqueue(); } else { result.setError(error.exception()); - transitionTo(ERROR); + transitionTo(ERROR, error.exception()); } if (error == Errors.NONE || !result.isSuccessful()) @@ -659,15 +688,15 @@ public void handleResponse(AbstractResponse response) { } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { log.error("Seems like the broker has bad transaction state. producerId: {}, error: {}. message: {}", pidAndEpoch.producerId, error, error.message()); - transitionTo(ERROR); + transitionTo(ERROR, error.exception()); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED); + transitionTo(FENCED, error.exception()); log.error("Epoch has become invalid: producerId: {}. epoch: {}. Message: {}", pidAndEpoch.producerId, pidAndEpoch.epoch, error.message()); } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) { - transitionTo(ERROR); + transitionTo(ERROR, error.exception()); log.error("No permissions add some partitions to the transaction: {}", error.message()); } else { - transitionTo(ERROR); + transitionTo(ERROR, error.exception()); log.error("Could not add partitions to transaction due to unknown error: {}", error.message()); } } @@ -702,11 +731,11 @@ public void handleResponse(AbstractResponse responseBody) { } else if (findCoordinatorResponse.error() == Errors.COORDINATOR_NOT_AVAILABLE) { reenqueue(); } else if (findCoordinatorResponse.error() == Errors.GROUP_AUTHORIZATION_FAILED) { - transitionTo(ERROR); + transitionTo(ERROR, findCoordinatorResponse.error().exception()); log.error("Not authorized to access the group with type {} and key {}. Message: {} ", type, coordinatorKey, findCoordinatorResponse.error().message()); } else { - transitionTo(ERROR); + transitionTo(ERROR, findCoordinatorResponse.error().exception()); log.error("Could not find a coordinator with type {} for unknown reasons. coordinatorKey: {}", type, coordinatorKey, findCoordinatorResponse.error().message()); } @@ -738,11 +767,11 @@ public void handleResponse(AbstractResponse responseBody) { } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED); + transitionTo(FENCED, error.exception()); result.setError(error.exception()); } else { result.setError(error.exception()); - transitionTo(ERROR); + transitionTo(ERROR, error.exception()); } if (error == Errors.NONE || !result.isSuccessful()) @@ -777,10 +806,10 @@ public void handleResponse(AbstractResponse responseBody) { } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED); + transitionTo(FENCED, error.exception()); result.setError(error.exception()); } else { - transitionTo(ERROR); + transitionTo(ERROR, error.exception()); result.setError(error.exception()); } @@ -819,12 +848,12 @@ public void handleResponse(AbstractResponse responseBody) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP, consumerGroupId); } } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED); + transitionTo(FENCED, error.exception()); result.setError(error.exception()); break; } else { result.setError(error.exception()); - transitionTo(ERROR); + transitionTo(ERROR, error.exception()); break; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index bda0fc0bb4bfd..cdb7893cd5c1e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -516,7 +516,8 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons } private void failBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, RuntimeException exception) { - if (transactionState != null && batch.producerId() == transactionState.pidAndEpoch().producerId) { + if (transactionState != null && !transactionState.isTransactional() + && batch.producerId() == transactionState.pidAndEpoch().producerId) { // Reset the transaction state since we have hit an irrecoverable exception and cannot make any guarantees // about the previously committed message. Note that this will discard the producer id and sequence // numbers for all existing partitions. diff --git a/gradle/findbugs-exclude.xml b/gradle/findbugs-exclude.xml index 4fc729e9e6168..eeea7fc04eef7 100644 --- a/gradle/findbugs-exclude.xml +++ b/gradle/findbugs-exclude.xml @@ -290,6 +290,15 @@ For a detailed description of findbugs bug categories, see http://findbugs.sourc + + + + + + + From cd0d4b8dc8a7e156e53960834df237ae8dcc09f3 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 25 Apr 2017 23:12:31 -0700 Subject: [PATCH 12/21] Minor cleanup --- .../clients/producer/TransactionState.java | 72 +++++++++---------- 1 file changed, 32 insertions(+), 40 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index 60a0c1a341384..7c420861a0b53 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -52,13 +52,6 @@ import java.util.PriorityQueue; import java.util.Set; -import static org.apache.kafka.clients.producer.TransactionState.State.ABORTING_TRANSACTION; -import static org.apache.kafka.clients.producer.TransactionState.State.COMMITTING_TRANSACTION; -import static org.apache.kafka.clients.producer.TransactionState.State.ERROR; -import static org.apache.kafka.clients.producer.TransactionState.State.FENCED; -import static org.apache.kafka.clients.producer.TransactionState.State.INITIALIZING; -import static org.apache.kafka.clients.producer.TransactionState.State.IN_TRANSACTION; -import static org.apache.kafka.clients.producer.TransactionState.State.READY; import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_EPOCH; import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_ID; @@ -86,7 +79,7 @@ public class TransactionState { private volatile State currentState = State.UNINITIALIZED; private Exception lastError = null; - public enum State { + private enum State { UNINITIALIZED, INITIALIZING, READY, @@ -240,7 +233,7 @@ public boolean isValid() { public synchronized FutureTransactionalResult initializeTransactions() { ensureTransactional(); - if (!transitionTo(INITIALIZING)) + if (!transitionTo(State.INITIALIZING)) throw new IllegalStateException("Could not initialize transactions. Either transactions have already been " + "initialized or are being initialized."); setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); @@ -258,16 +251,15 @@ public synchronized FutureTransactionalResult initializeTransactions() { return resultFuture; } - public synchronized void beginTransaction() { ensureTransactional(); - if (!transitionTo(IN_TRANSACTION)) + if (!transitionTo(State.IN_TRANSACTION)) throw new IllegalStateException("Producer isn't ready to begin a transaction."); } public synchronized FutureTransactionalResult beginCommittingTransaction() { ensureTransactional(); - if (!transitionTo(COMMITTING_TRANSACTION)) { + if (!transitionTo(State.COMMITTING_TRANSACTION)) { String msg = "Cannot commit transaction, either because a transaction is already " + "being completed at the moment, or because there has been an error with a previous request."; if (lastError != null) @@ -281,7 +273,7 @@ public synchronized FutureTransactionalResult beginCommittingTransaction() { public synchronized FutureTransactionalResult beginAbortingTransaction() { ensureTransactional(); - if (!transitionTo(ABORTING_TRANSACTION)) { + if (!transitionTo(State.ABORTING_TRANSACTION)) { String msg = "Cannot abort transaction, either because a transaction is already " + "being completed at the moment, or because there has been an error with a previous request."; if (lastError != null) @@ -307,7 +299,7 @@ private FutureTransactionalResult beginCompletingTransaction(boolean isCommit) { public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map offsets, String consumerGroupId) { ensureTransactional(); - if (currentState != IN_TRANSACTION) { + if (currentState != State.IN_TRANSACTION) { String msg = "Cannot send offsets to transaction either because the producer is not in an " + "active transaction or because there has been an error with one or more previous requests."; if (lastError != null) @@ -420,19 +412,19 @@ public boolean isTransactional() { } public boolean isFenced() { - return currentState == FENCED; + return currentState == State.FENCED; } public boolean isCompletingTransaction() { - return currentState == COMMITTING_TRANSACTION || currentState == ABORTING_TRANSACTION; + return currentState == State.COMMITTING_TRANSACTION || currentState == State.ABORTING_TRANSACTION; } public boolean isInTransaction() { - return currentState == IN_TRANSACTION || isCompletingTransaction(); + return currentState == State.IN_TRANSACTION || isCompletingTransaction(); } public boolean isInErrorState() { - return currentState == ERROR; + return currentState == State.ERROR; } @@ -463,9 +455,9 @@ public void needsCoordinator(TransactionalRequest request) { public synchronized boolean maybeSetError(Exception exception) { if (isTransactional() && isInTransaction()) { if (exception instanceof ProducerFencedException) - transitionTo(FENCED, exception); + transitionTo(State.FENCED, exception); else - transitionTo(ERROR, exception); + transitionTo(State.ERROR, exception); return true; } return false; @@ -488,7 +480,7 @@ private boolean transitionTo(State target) { } private boolean transitionTo(State target, Exception error) { - if (target == ERROR && error != null) + if (target == State.ERROR && error != null) lastError = error; if (currentState.isTransitionValid(currentState, target)) { currentState = target; @@ -530,11 +522,11 @@ public boolean hasPendingOffsetCommits() { // visible for testing public boolean isReadyForTransaction() { - return isTransactional() && currentState == READY; + return isTransactional() && currentState == State.READY; } private void completeTransaction() { - transitionTo(READY); + transitionTo(State.READY); lastError = null; partitionsInTransaction.clear(); } @@ -603,7 +595,7 @@ private abstract class TransactionalRequestCallBack implements RequestCompletion public void onComplete(ClientResponse response) { if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) { log.error("Detected more than one inflight transactional request. This should never happen."); - transitionTo(ERROR); + transitionTo(State.ERROR); } resetInFlightRequestCorrelationId(); @@ -615,7 +607,7 @@ public void onComplete(ClientResponse response) { result.done(); } log.error("Could not execute transactional request because the broker isn't on the right version."); - transitionTo(ERROR, Errors.UNSUPPORTED_VERSION.exception()); + transitionTo(State.ERROR, Errors.UNSUPPORTED_VERSION.exception()); } else if (response.hasResponse()) { handleResponse(response.responseBody()); } else { @@ -624,7 +616,7 @@ public void onComplete(ClientResponse response) { result.done(); } log.error("Could not execute transactional request for unknown reasons"); - transitionTo(ERROR, Errors.UNKNOWN.exception()); + transitionTo(State.ERROR, Errors.UNKNOWN.exception()); } } @@ -645,7 +637,7 @@ public void handleResponse(AbstractResponse responseBody) { Errors error = initPidResponse.error(); if (error == Errors.NONE) { setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch()); - transitionTo(READY); + transitionTo(State.READY); lastError = null; } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); @@ -654,7 +646,7 @@ public void handleResponse(AbstractResponse responseBody) { reenqueue(); } else { result.setError(error.exception()); - transitionTo(ERROR, error.exception()); + transitionTo(State.ERROR, error.exception()); } if (error == Errors.NONE || !result.isSuccessful()) @@ -688,15 +680,15 @@ public void handleResponse(AbstractResponse response) { } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { log.error("Seems like the broker has bad transaction state. producerId: {}, error: {}. message: {}", pidAndEpoch.producerId, error, error.message()); - transitionTo(ERROR, error.exception()); + transitionTo(State.ERROR, error.exception()); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED, error.exception()); + transitionTo(State.FENCED, error.exception()); log.error("Epoch has become invalid: producerId: {}. epoch: {}. Message: {}", pidAndEpoch.producerId, pidAndEpoch.epoch, error.message()); } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) { - transitionTo(ERROR, error.exception()); + transitionTo(State.ERROR, error.exception()); log.error("No permissions add some partitions to the transaction: {}", error.message()); } else { - transitionTo(ERROR, error.exception()); + transitionTo(State.ERROR, error.exception()); log.error("Could not add partitions to transaction due to unknown error: {}", error.message()); } } @@ -731,11 +723,11 @@ public void handleResponse(AbstractResponse responseBody) { } else if (findCoordinatorResponse.error() == Errors.COORDINATOR_NOT_AVAILABLE) { reenqueue(); } else if (findCoordinatorResponse.error() == Errors.GROUP_AUTHORIZATION_FAILED) { - transitionTo(ERROR, findCoordinatorResponse.error().exception()); + transitionTo(State.ERROR, findCoordinatorResponse.error().exception()); log.error("Not authorized to access the group with type {} and key {}. Message: {} ", type, coordinatorKey, findCoordinatorResponse.error().message()); } else { - transitionTo(ERROR, findCoordinatorResponse.error().exception()); + transitionTo(State.ERROR, findCoordinatorResponse.error().exception()); log.error("Could not find a coordinator with type {} for unknown reasons. coordinatorKey: {}", type, coordinatorKey, findCoordinatorResponse.error().message()); } @@ -767,11 +759,11 @@ public void handleResponse(AbstractResponse responseBody) { } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED, error.exception()); + transitionTo(State.FENCED, error.exception()); result.setError(error.exception()); } else { result.setError(error.exception()); - transitionTo(ERROR, error.exception()); + transitionTo(State.ERROR, error.exception()); } if (error == Errors.NONE || !result.isSuccessful()) @@ -806,10 +798,10 @@ public void handleResponse(AbstractResponse responseBody) { } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { reenqueue(); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED, error.exception()); + transitionTo(State.FENCED, error.exception()); result.setError(error.exception()); } else { - transitionTo(ERROR, error.exception()); + transitionTo(State.ERROR, error.exception()); result.setError(error.exception()); } @@ -848,12 +840,12 @@ public void handleResponse(AbstractResponse responseBody) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP, consumerGroupId); } } else if (error == Errors.INVALID_PRODUCER_EPOCH) { - transitionTo(FENCED, error.exception()); + transitionTo(State.FENCED, error.exception()); result.setError(error.exception()); break; } else { result.setError(error.exception()); - transitionTo(ERROR, error.exception()); + transitionTo(State.ERROR, error.exception()); break; } } From b01de617d13bde8e2fc66f8f8b7ec701ce2731bb Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 26 Apr 2017 14:58:21 -0700 Subject: [PATCH 13/21] Addressed more comments (online and offline): 1. Changed the 'IllegalStateException' to 'KafkaException' 2. Wake up the sender on every transactional request 3. Tightened up the protection against errors. Once we are in an error state, no transactional send or transactional rpc will be allowed until abort is called. 4. We only allow initTransactions from an uninitialized or error state. --- .../clients/consumer/ConsumerConfig.java | 14 +++-- .../clients/consumer/internals/Fetcher.java | 7 ++- .../kafka/clients/producer/KafkaProducer.java | 50 ++++++++++----- .../kafka/clients/producer/Producer.java | 39 ++---------- .../clients/producer/ProducerConfig.java | 1 + .../clients/producer/TransactionState.java | 63 ++++++++++--------- .../producer/internals/ProducerBatch.java | 4 ++ .../clients/producer/internals/Sender.java | 9 +-- .../apache/kafka/common/config/ConfigDef.java | 3 +- .../common/record/MemoryRecordsBuilder.java | 4 ++ .../common/requests/ListOffsetRequest.java | 1 - .../consumer/internals/FetcherTest.java | 44 +++++++++++-- .../producer/internals/TransactionsTest.java | 4 +- 13 files changed, 143 insertions(+), 100 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index 8643d07454f69..0c0c29da20c4a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -237,15 +238,15 @@ public class ConsumerConfig extends AbstractConfig { /** isolation.level */ public static final String ISOLATION_LEVEL_CONFIG = "isolation.level"; - public static final String ISOLATION_LEVEL_DOC = "

Controls how to read messages written transactionally. If set to READ_COMMITTED, consumer.poll() will only return" + - " transactional messages which have been committed. If set to READ_UNCOMMITTED' (the default), consumer.poll() will return all messages, even transactional messages" + + public static final String ISOLATION_LEVEL_DOC = "

Controls how to read messages written transactionally. If set to read_committed, consumer.poll() will only return" + + " transactional messages which have been committed. If set to read_uncommitted' (the default), consumer.poll() will return all messages, even transactional messages" + " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.

Messages will always be returned in offset order. Hence, in " + - " READ_COMMITTED mode, consumer.poll() will only return messages up to the last stable offset (LSO), which is the one less than the offset of the first open transaction." + - " In particular any messages appearing after messages belonging to ongoing transactions will be withheld until the relevant transaction has been completed. As a result, READ_COMMITTED" + - " consumers will not be able to read up to the high watermark when there are in flight transactions.

Further, when in READ_COMMITTED the seekToEnd method will" + + " read_committed mode, consumer.poll() will only return messages up to the last stable offset (LSO), which is the one less than the offset of the first open transaction." + + " In particular any messages appearing after messages belonging to ongoing transactions will be withheld until the relevant transaction has been completed. As a result, read_committed" + + " consumers will not be able to read up to the high watermark when there are in flight transactions.

Further, when in read_committed the seekToEnd method will" + " return the LSO"; - public static final String DEFAULT_ISOLATION_LEVEL = IsolationLevel.READ_UNCOMMITTED.toString(); + public static final String DEFAULT_ISOLATION_LEVEL = IsolationLevel.READ_UNCOMMITTED.toString().toLowerCase(Locale.ROOT); static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, @@ -421,6 +422,7 @@ public class ConsumerConfig extends AbstractConfig { .define(ISOLATION_LEVEL_CONFIG, Type.STRING, DEFAULT_ISOLATION_LEVEL, + in(IsolationLevel.READ_COMMITTED.toString().toLowerCase(Locale.ROOT), IsolationLevel.READ_UNCOMMITTED.toString().toLowerCase(Locale.ROOT)), Importance.MEDIUM, ISOLATION_LEVEL_DOC) // security support diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 58e07be6b0ba7..947214f2e7601 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -472,7 +472,7 @@ public Map endOffsets(Collection partition } private long endTimestamp() { - return isolationLevel == IsolationLevel.READ_COMMITTED ? ListOffsetRequest.LSO_TIMESTAMP : ListOffsetRequest.LATEST_TIMESTAMP; + return ListOffsetRequest.LATEST_TIMESTAMP; } private Map beginningOrEndOffset(Collection partitions, @@ -999,6 +999,7 @@ private Record nextFetchedRecord() { maybeEnsureValid(currentBatch); if (isolationLevel == IsolationLevel.READ_COMMITTED && isBatchAborted(currentBatch)) { + nextFetchOffset = currentBatch.lastOffset() + 1; continue; } @@ -1056,7 +1057,7 @@ private boolean isBatchAborted(RecordBatch batch) { abortedProducerIds.add(batch.producerId()); abortedTransactions.poll(); } - log.trace("Skipping aborted record batch with producerId {} and base offset {}", batch.producerId(), batch.baseOffset()); + log.trace("Skipping aborted record batch with producerId {} and base offset {}, partition: {}", batch.producerId(), batch.baseOffset(), partition); return true; } return false; @@ -1086,7 +1087,7 @@ private boolean containsAbortMarker(RecordBatch batch) { Record firstRecord = batchIterator.hasNext() ? batchIterator.next() : null; boolean containsAbortMarker = firstRecord != null && firstRecord.isControlRecord() && ControlRecordType.ABORT == ControlRecordType.parse(firstRecord.key()); if (containsAbortMarker && batchIterator.hasNext()) - throw new CorruptRecordException("A record batch containing a control message contained more than one record."); + throw new CorruptRecordException("A record batch containing a control message contained more than one record. partition: " + partition + ", offset: " + batch.baseOffset()); return containsAbortMarker; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index d61246af547f7..7c12a966f1e85 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.internals.FutureTransactionalResult; import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.Sender; @@ -473,7 +474,6 @@ private static int parseAcks(String acksString) { public void initTransactions() { if (transactionState == null) throw new IllegalStateException("Cannot call initTransactions without setting a transactional id."); - transactionState.initializeTransactions().get(); } @@ -505,7 +505,9 @@ public void sendOffsetsToTransaction(Map offs String consumerGroupId) throws ProducerFencedException { if (transactionState == null) throw new IllegalStateException("Cannot send offsets to transaction since transactions are not enabled."); - transactionState.sendOffsetsToTransaction(offsets, consumerGroupId).get(); + FutureTransactionalResult result = transactionState.sendOffsetsToTransaction(offsets, consumerGroupId); + sender.wakeup(); + result.get(); } /** @@ -517,7 +519,9 @@ public void sendOffsetsToTransaction(Map offs public void commitTransaction() throws ProducerFencedException { if (transactionState == null) throw new IllegalStateException("Cannot commit transaction since transactions are not enabled"); - transactionState.beginCommittingTransaction().get(); + FutureTransactionalResult result = transactionState.beginCommittingTransaction(); + sender.wakeup(); + result.get(); } /** @@ -529,7 +533,9 @@ public void commitTransaction() throws ProducerFencedException { public void abortTransaction() throws ProducerFencedException { if (transactionState == null) throw new IllegalStateException("Cannot abort transaction since transactions are not enabled."); - transactionState.beginAbortingTransaction().get(); + FutureTransactionalResult result = transactionState.beginAbortingTransaction(); + sender.wakeup(); + result.get(); } /** @@ -625,17 +631,7 @@ public Future send(ProducerRecord record, Callback callbac * Implementation of asynchronously send a record to a topic. */ private Future doSend(ProducerRecord record, Callback callback) { - if (transactionState != null) { - if (transactionState.isTransactional() && !transactionState.hasPid()) - throw new IllegalStateException("Cannot perform a 'send' before completing a call to initTransactions when transactions are enabled."); - - if (transactionState.isFenced() && transactionState.isInTransaction()) - throw new ProducerFencedException("The current producer has been fenced off by a another producer using the same transactional id."); - - if (transactionState.isCompletingTransaction()) - throw new IllegalStateException("Cannot call send while a commit or abort is in progress."); - } - + ensureProperTransactionalState(); TopicPartition tp = null; try { // first make sure the metadata for the topic is available @@ -716,6 +712,30 @@ private Future doSend(ProducerRecord record, Callback call } } + private void ensureProperTransactionalState() { + if (transactionState == null) + return; + + if (transactionState.isTransactional() && !transactionState.hasPid()) + throw new IllegalStateException("Cannot perform a 'send' before completing a call to initTransactions when transactions are enabled."); + + if (transactionState.isFenced()) + throw new ProducerFencedException("The current producer has been fenced off by a another producer using the same transactional id."); + + if (transactionState.isInTransaction()) { + if (transactionState.isInErrorState()) { + String errorMessage = "Cannot perform a transactional send because at least one previous transactional request has failed with errors."; + Exception lastError = transactionState.lastError(); + if (lastError != null) + throw new KafkaException(errorMessage, lastError); + else + throw new KafkaException(errorMessage); + } + if (transactionState.isCompletingTransaction()) + throw new IllegalStateException("Cannot call send while a commit or abort is in progress."); + } + } + /** * Wait for cluster metadata including partitions for the given topic to be available. * @param topic The topic we want metadata for diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java index a057de9f50faf..1e77633506dfc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java @@ -37,57 +37,28 @@ public interface Producer extends Closeable { /** - * Needs to be called before any of the other transaction methods. Assumes that - * the transactional.id is specified in the producer configuration. - * - * This method does the following: - * 1. Ensures any transactions initiated by previous instances of the producer - * are completed. If the previous instance had failed with a transaction in - * progress, it will be aborted. If the last transaction had begun completion, - * but not yet finished, this method awaits its completion. - * 2. Gets the internal producer id and epoch, used in all future transactional - * messages issued by the producer. - * - * @throws IllegalStateException if the TransactionalId for the producer is not set - * in the configuration. + * See {@link KafkaProducer#initTransactions()} */ void initTransactions(); /** - * Should be called before the start of each new transaction. - * - * @throws ProducerFencedException if another producer with the same - * transactional.id is active. + * See {@link KafkaProducer#beginTransaction()} */ void beginTransaction() throws ProducerFencedException; /** - * Sends a list of consumed offsets to the consumer group coordinator, and also marks - * those offsets as part of the current transaction. These offsets will be considered - * consumed only if the transaction is committed successfully. - * - * This method should be used when you need to batch consumed and produced messages - * together, typically in a consume-transform-produce pattern. - * - * @throws ProducerFencedException if another producer with the same - * transactional.id is active. + * See {@link KafkaProducer#sendOffsetsToTransaction(Map, String)} */ void sendOffsetsToTransaction(Map offsets, String consumerGroupId) throws ProducerFencedException; /** - * Commits the ongoing transaction. - * - * @throws ProducerFencedException if another producer with the same - * transactional.id is active. + * See {@link KafkaProducer#commitTransaction()} */ void commitTransaction() throws ProducerFencedException; /** - * Aborts the ongoing transaction. - * - * @throws ProducerFencedException if another producer with the same - * transactional.id is active. + * See {@link KafkaProducer#abortTransaction()} */ void abortTransaction() throws ProducerFencedException; diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index b54b589ec9371..4bceb9528b9f0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -346,6 +346,7 @@ public class ProducerConfig extends AbstractConfig { .define(TRANSACTIONAL_ID_CONFIG, Type.STRING, null, + new ConfigDef.NonEmptyString(), Importance.LOW, TRANSACTIONAL_ID_DOC); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java index 7c420861a0b53..0224df81ced2d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.internals.FutureTransactionalResult; import org.apache.kafka.clients.producer.internals.TransactionalRequestResult; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.ProducerFencedException; @@ -92,7 +93,7 @@ private enum State { private boolean isTransitionValid(State source, State target) { switch (target) { case INITIALIZING: - return source == UNINITIALIZED || source == READY || source == ERROR; + return source == UNINITIALIZED || source == ERROR; case READY: return source == INITIALIZING || source == COMMITTING_TRANSACTION || source == ABORTING_TRANSACTION; case IN_TRANSACTION: @@ -234,7 +235,7 @@ public boolean isValid() { public synchronized FutureTransactionalResult initializeTransactions() { ensureTransactional(); if (!transitionTo(State.INITIALIZING)) - throw new IllegalStateException("Could not initialize transactions. Either transactions have already been " + + throw new KafkaException("Could not initialize transactions. Either transactions have already been " + "initialized or are being initialized."); setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); this.sequenceNumbers.clear(); @@ -253,35 +254,27 @@ public synchronized FutureTransactionalResult initializeTransactions() { public synchronized void beginTransaction() { ensureTransactional(); + maybeFailWithError(); if (!transitionTo(State.IN_TRANSACTION)) - throw new IllegalStateException("Producer isn't ready to begin a transaction."); + throw new KafkaException("Producer isn't ready to begin a transaction, most likely because there is " + + "already an ongoing transaction."); } public synchronized FutureTransactionalResult beginCommittingTransaction() { ensureTransactional(); - if (!transitionTo(State.COMMITTING_TRANSACTION)) { - String msg = "Cannot commit transaction, either because a transaction is already " + - "being completed at the moment, or because there has been an error with a previous request."; - if (lastError != null) - throw new IllegalStateException(msg, lastError); - else - throw new IllegalStateException(msg); - } - + maybeFailWithError(); + if (!transitionTo(State.COMMITTING_TRANSACTION)) + throw new KafkaException("Cannot commit transaction, most likely because a transaction is already being completed."); return beginCompletingTransaction(true); } public synchronized FutureTransactionalResult beginAbortingTransaction() { ensureTransactional(); - if (!transitionTo(State.ABORTING_TRANSACTION)) { - String msg = "Cannot abort transaction, either because a transaction is already " + - "being completed at the moment, or because there has been an error with a previous request."; - if (lastError != null) - throw new IllegalStateException(msg, lastError); - else - throw new IllegalStateException(msg); - } - + if (isFenced()) + throw new ProducerFencedException("There is a newer producer using the same transactional.id."); + if (!transitionTo(State.ABORTING_TRANSACTION)) + throw new KafkaException("Cannot abort transaction, either because a transaction is already " + + "being completed at the moment, or because there has been an error with a previous request."); return beginCompletingTransaction(false); } @@ -299,14 +292,10 @@ private FutureTransactionalResult beginCompletingTransaction(boolean isCommit) { public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map offsets, String consumerGroupId) { ensureTransactional(); - if (currentState != State.IN_TRANSACTION) { - String msg = "Cannot send offsets to transaction either because the producer is not in an " + - "active transaction or because there has been an error with one or more previous requests."; - if (lastError != null) - throw new IllegalStateException(msg, lastError); - else - throw new IllegalStateException(msg); - } + maybeFailWithError(); + if (currentState != State.IN_TRANSACTION) + throw new KafkaException("Cannot send offsets to transaction either because the producer is not in an " + + "active transaction or because there has been an error with one or more previous requests."); TransactionalRequestResult result = new TransactionalRequestResult(); FutureTransactionalResult resultFuture = new FutureTransactionalResult(result); @@ -399,6 +388,10 @@ public TransactionalRequest nextTransactionalRequest() { return pendingTransactionalRequests.poll(); } + public Exception lastError() { + return lastError; + } + public String transactionalId() { return transactionalId; } @@ -496,6 +489,18 @@ private void ensureTransactional() { throw new IllegalStateException("Transactional method invoked on a non-transactional producer."); } + private void maybeFailWithError() { + if (isFenced()) + throw new ProducerFencedException("There is a newer producer instance using the same transactional id."); + if (isInErrorState()) { + String errorMessage = "Cannot execute transactional method because we are in an error state."; + if (lastError != null) + throw new KafkaException(errorMessage, lastError); + else + throw new KafkaException(errorMessage); + } + } + private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) { switch (type) { case GROUP: diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index bacf0a24b0984..ced875d42b909 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -273,4 +273,8 @@ public byte magic() { public long producerId() { return recordsBuilder.producerId(); } + + public short producerEpoch() { + return recordsBuilder.producerEpoch(); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index cdb7893cd5c1e..68820ee2a8248 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -24,11 +24,11 @@ import org.apache.kafka.clients.RequestCompletionHandler; import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InvalidMetadataException; -import org.apache.kafka.common.errors.InvalidTxnStateException; import org.apache.kafka.common.errors.OutOfOrderSequenceException; import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TopicAuthorizationException; @@ -291,7 +291,7 @@ private boolean maybeSendTransactionalRequest(long now) { } if (nextRequest.isEndTxnRequest() && transactionState.isInErrorState()) { - nextRequest.maybeTerminateWithError(new InvalidTxnStateException("Cannot commit transaction when there are " + + nextRequest.maybeTerminateWithError(new KafkaException("Cannot commit transaction when there are " + "request errors. Please check your logs for the details of the errors encountered.")); return false; } @@ -458,7 +458,7 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons error); if (transactionState == null) { reenqueueBatch(batch, now); - } else if (transactionState.pidAndEpoch().producerId == batch.producerId()) { + } else if (transactionState.pidAndEpoch().producerId == batch.producerId() && transactionState.pidAndEpoch().epoch == batch.producerEpoch()) { // If idempotence is enabled only retry the request if the current PID is the same as the pid of the batch. log.debug("Retrying batch to topic-partition {}. Sequence number : {}", batch.topicPartition, transactionState.sequenceNumber(batch.topicPartition)); @@ -493,7 +493,8 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons } else { completeBatch(batch, response); - if (transactionState != null && transactionState.pidAndEpoch().producerId == batch.producerId()) { + if (transactionState != null && transactionState.pidAndEpoch().producerId == batch.producerId() + && transactionState.pidAndEpoch().epoch == batch.producerEpoch()) { transactionState.incrementSequenceNumber(batch.topicPartition, batch.recordCount); log.debug("Incremented sequence number for topic-partition {} to {}", batch.topicPartition, transactionState.sequenceNumber(batch.topicPartition)); diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java index b3f6ce067b0a9..5afc9cf31a40e 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java +++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java @@ -900,11 +900,10 @@ public String toString() { } public static class NonEmptyString implements Validator { - @Override public void ensureValid(String name, Object o) { String s = (String) o; - if (s.isEmpty()) { + if (s != null && s.isEmpty()) { throw new ConfigException(name, o, "String must be non-empty"); } } diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index c8682cd5d2839..eb521346ff329 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -660,4 +660,8 @@ public RecordsInfo(long maxTimestamp, public long producerId() { return this.producerId; } + + public short producerEpoch() { + return this.producerEpoch; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java index feb3635b2f52d..33270717ef26e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java @@ -33,7 +33,6 @@ import java.util.Set; public class ListOffsetRequest extends AbstractRequest { - public static final long LSO_TIMESTAMP = -3L; public static final long EARLIEST_TIMESTAMP = -2L; public static final long LATEST_TIMESTAMP = -1L; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 2aa6336c05aa4..6059180bb416a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1332,9 +1332,45 @@ public void testReturnAbortedTransactionsinUncommittedMode() { assertTrue(fetchedRecords.containsKey(tp1)); } - private int appendTransactionalRecords(ByteBuffer buffer, long pid, int baseOffset, SimpleRecord... records) { + @Test + public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { + Fetcher fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + long currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds()); + buffer.flip(); + + List abortedTransactions = new ArrayList<>(); + abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + subscriptions.assignFromUser(singleton(tp1)); + + subscriptions.seek(tp1, 0); + + // normal fetch + assertEquals(1, fetcher.sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(0); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetcher.fetchedRecords(); + + // Ensure that we don't return any of the aborted records, but yet advance the consumer position. + assertFalse(fetchedRecords.containsKey(tp1)); + assertEquals(currentOffset, (long) subscriptions.position(tp1)); + } + + private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOffset, SimpleRecord... records) { MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, - TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, (int) baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); for (SimpleRecord record : records) { builder.append(record); @@ -1351,9 +1387,9 @@ private int commitTransaction(ByteBuffer buffer, long pid, int baseOffset, long return 1; } - private int abortTransaction(ByteBuffer buffer, long pid, int baseOffset, long timestamp) { + private int abortTransaction(ByteBuffer buffer, long pid, long baseOffset, long timestamp) { MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, - TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); + TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, (int) baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); builder.appendControlRecord(timestamp, ControlRecordType.ABORT, null); builder.build(); return 1; diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java index 15f1c392b8b1f..f8ecc1d82c6a3 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java @@ -24,9 +24,9 @@ import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.InvalidTxnStateException; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -455,7 +455,7 @@ public void testDisallowCommitOnProduceFailure() throws InterruptedException { commitResult.get(); fail(); // the get() must throw an exception. } catch (RuntimeException e) { - assertTrue(e instanceof InvalidTxnStateException); + assertTrue(e instanceof KafkaException); } // Commit is not allowed, so let's abort and try again. From 06cc10a92a4ce11fd45e2d7658b58f8c5c245b21 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 26 Apr 2017 15:09:56 -0700 Subject: [PATCH 14/21] Renamed TransactionState to TransactionManager. Moved TransactionManager to the internals package. --- checkstyle/suppressions.xml | 6 +- .../kafka/clients/producer/KafkaProducer.java | 75 ++++---- .../producer/internals/ProducerBatch.java | 3 +- .../producer/internals/RecordAccumulator.java | 23 ++- .../clients/producer/internals/Sender.java | 67 ++++--- .../TransactionManager.java} | 18 +- .../internals/RecordAccumulatorTest.java | 3 +- .../producer/internals/SenderTest.java | 49 +++-- ...sTest.java => TransactionManagerTest.java} | 171 ++++++++++-------- .../internals/TransactionStateTest.java | 60 ------ 10 files changed, 216 insertions(+), 259 deletions(-) rename clients/src/main/java/org/apache/kafka/clients/producer/{TransactionState.java => internals/TransactionManager.java} (98%) rename clients/src/test/java/org/apache/kafka/clients/producer/internals/{TransactionsTest.java => TransactionManagerTest.java} (78%) delete mode 100644 clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index b90b7948a774b..eae8dde29cc84 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -8,7 +8,7 @@ + files="(Fetcher|ConsumerCoordinator|KafkaConsumer|KafkaProducer|SaslServerAuthenticator|Utils|TransactionManagerTest).java"/> + files="(KafkaConsumer|ConsumerCoordinator|Fetcher|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager).java"/> @@ -53,7 +53,7 @@ + files="(Sender|Fetcher|KafkaConsumer|Metrics|ConsumerCoordinator|RequestResponse|TransactionManager)Test.java"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 7c12a966f1e85..1045f4c4798a3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -25,6 +25,7 @@ import org.apache.kafka.clients.producer.internals.ProducerInterceptors; import org.apache.kafka.clients.producer.internals.RecordAccumulator; import org.apache.kafka.clients.producer.internals.Sender; +import org.apache.kafka.clients.producer.internals.TransactionManager; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -162,7 +163,7 @@ public class KafkaProducer implements Producer { private final int requestTimeoutMs; private final ProducerInterceptors interceptors; private final ApiVersions apiVersions; - private final TransactionState transactionState; + private final TransactionManager transactionManager; /** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings @@ -266,10 +267,10 @@ private KafkaProducer(ProducerConfig config, Serializer keySerializer, Serial this.maxBlockTimeMs = configureMaxBlockTime(config, userProvidedConfigs); this.requestTimeoutMs = configureRequestTimeout(config, userProvidedConfigs); - this.transactionState = configureTransactionState(config); - int retries = configureRetries(config, transactionState != null); - int maxInflightRequests = configureInflightRequests(config, transactionState != null); - short acks = configureAcks(config, transactionState != null); + this.transactionManager = configureTransactionState(config); + int retries = configureRetries(config, transactionManager != null); + int maxInflightRequests = configureInflightRequests(config, transactionManager != null); + short acks = configureAcks(config, transactionManager != null); this.apiVersions = new ApiVersions(); this.accumulator = new RecordAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), @@ -280,7 +281,7 @@ private KafkaProducer(ProducerConfig config, Serializer keySerializer, Serial metrics, time, apiVersions, - transactionState); + transactionManager); List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)); this.metadata.update(Cluster.bootstrap(addresses), Collections.emptySet(), time.milliseconds()); ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config); @@ -308,7 +309,7 @@ private KafkaProducer(ProducerConfig config, Serializer keySerializer, Serial Time.SYSTEM, this.requestTimeoutMs, config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG), - this.transactionState, + this.transactionManager, apiVersions); String ioThreadName = "kafka-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : ""); this.ioThread = new KafkaThread(ioThreadName, this.sender, true); @@ -366,9 +367,9 @@ private static int configureRequestTimeout(ProducerConfig config, Map offsets, String consumerGroupId) throws ProducerFencedException { - if (transactionState == null) + if (transactionManager == null) throw new IllegalStateException("Cannot send offsets to transaction since transactions are not enabled."); - FutureTransactionalResult result = transactionState.sendOffsetsToTransaction(offsets, consumerGroupId); + FutureTransactionalResult result = transactionManager.sendOffsetsToTransaction(offsets, consumerGroupId); sender.wakeup(); result.get(); } @@ -517,9 +518,9 @@ public void sendOffsetsToTransaction(Map offs * transactional.id is active. */ public void commitTransaction() throws ProducerFencedException { - if (transactionState == null) + if (transactionManager == null) throw new IllegalStateException("Cannot commit transaction since transactions are not enabled"); - FutureTransactionalResult result = transactionState.beginCommittingTransaction(); + FutureTransactionalResult result = transactionManager.beginCommittingTransaction(); sender.wakeup(); result.get(); } @@ -531,9 +532,9 @@ public void commitTransaction() throws ProducerFencedException { * transactional.id is active. */ public void abortTransaction() throws ProducerFencedException { - if (transactionState == null) + if (transactionManager == null) throw new IllegalStateException("Cannot abort transaction since transactions are not enabled."); - FutureTransactionalResult result = transactionState.beginAbortingTransaction(); + FutureTransactionalResult result = transactionManager.beginAbortingTransaction(); sender.wakeup(); result.get(); } @@ -663,10 +664,10 @@ private Future doSend(ProducerRecord record, Callback call long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp(); log.trace("Sending record {} with callback {} to topic {} partition {}", record, callback, record.topic(), partition); // producer callback will make sure to call both 'callback' and interceptor callback - Callback interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp, transactionState); + Callback interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp, transactionManager); - if (transactionState != null && transactionState.isInTransaction()) { - transactionState.maybeAddPartitionToTransaction(new TopicPartition(record.topic(), record.partition())); + if (transactionManager != null && transactionManager.isInTransaction()) { + transactionManager.maybeAddPartitionToTransaction(new TopicPartition(record.topic(), record.partition())); } @@ -713,25 +714,25 @@ private Future doSend(ProducerRecord record, Callback call } private void ensureProperTransactionalState() { - if (transactionState == null) + if (transactionManager == null) return; - if (transactionState.isTransactional() && !transactionState.hasPid()) + if (transactionManager.isTransactional() && !transactionManager.hasPid()) throw new IllegalStateException("Cannot perform a 'send' before completing a call to initTransactions when transactions are enabled."); - if (transactionState.isFenced()) + if (transactionManager.isFenced()) throw new ProducerFencedException("The current producer has been fenced off by a another producer using the same transactional id."); - if (transactionState.isInTransaction()) { - if (transactionState.isInErrorState()) { + if (transactionManager.isInTransaction()) { + if (transactionManager.isInErrorState()) { String errorMessage = "Cannot perform a transactional send because at least one previous transactional request has failed with errors."; - Exception lastError = transactionState.lastError(); + Exception lastError = transactionManager.lastError(); if (lastError != null) throw new KafkaException(errorMessage, lastError); else throw new KafkaException(errorMessage); } - if (transactionState.isCompletingTransaction()) + if (transactionManager.isCompletingTransaction()) throw new IllegalStateException("Cannot call send while a commit or abort is in progress."); } } @@ -1029,14 +1030,14 @@ private static class InterceptorCallback implements Callback { private final Callback userCallback; private final ProducerInterceptors interceptors; private final TopicPartition tp; - private final TransactionState transactionState; + private final TransactionManager transactionManager; public InterceptorCallback(Callback userCallback, ProducerInterceptors interceptors, - TopicPartition tp, TransactionState transactionState) { + TopicPartition tp, TransactionManager transactionManager) { this.userCallback = userCallback; this.interceptors = interceptors; this.tp = tp; - this.transactionState = transactionState; + this.transactionManager = transactionManager; } public void onCompletion(RecordMetadata metadata, Exception exception) { @@ -1051,8 +1052,8 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { if (this.userCallback != null) this.userCallback.onCompletion(metadata, exception); - if (exception != null && transactionState != null) - transactionState.maybeSetError(exception); + if (exception != null && transactionManager != null) + transactionManager.maybeSetError(exception); } } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index ced875d42b909..eba307879f548 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -18,7 +18,6 @@ import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.record.AbstractRecords; @@ -231,7 +230,7 @@ public boolean isFull() { return recordsBuilder.isFull(); } - public void setProducerState(TransactionState.PidAndEpoch pidAndEpoch, int baseSequence) { + public void setProducerState(TransactionManager.PidAndEpoch pidAndEpoch, int baseSequence) { recordsBuilder.setProducerState(pidAndEpoch.producerId, pidAndEpoch.epoch, baseSequence); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 72fcd9fc1345a..aa4d9d36b8e3f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -18,7 +18,6 @@ import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.Node; @@ -82,7 +81,7 @@ public final class RecordAccumulator { // The following variables are only accessed by the sender thread, so we don't need to protect them. private final Set muted; private int drainIndex; - private final TransactionState transactionState; + private final TransactionManager transactionManager; /** * Create a new record accumulator @@ -98,7 +97,7 @@ public final class RecordAccumulator { * @param metrics The metrics * @param time The time instance to use * @param apiVersions Request API versions for current connected brokers - * @param transactionState The shared transaction state object which tracks Pids, epochs, and sequence numbers per + * @param transactionManager The shared transaction state object which tracks Pids, epochs, and sequence numbers per * partition. */ public RecordAccumulator(int batchSize, @@ -109,7 +108,7 @@ public RecordAccumulator(int batchSize, Metrics metrics, Time time, ApiVersions apiVersions, - TransactionState transactionState) { + TransactionManager transactionManager) { this.drainIndex = 0; this.closed = false; this.flushesInProgress = new AtomicInteger(0); @@ -125,7 +124,7 @@ public RecordAccumulator(int batchSize, this.muted = new HashSet<>(); this.time = time; this.apiVersions = apiVersions; - this.transactionState = transactionState; + this.transactionManager = transactionManager; registerMetrics(metrics, metricGrpName); } @@ -229,13 +228,13 @@ public RecordAppendResult append(TopicPartition tp, } private MemoryRecordsBuilder recordsBuilder(ByteBuffer buffer, byte maxUsableMagic) { - if (transactionState != null && maxUsableMagic < RecordBatch.MAGIC_VALUE_V2) { + if (transactionManager != null && maxUsableMagic < RecordBatch.MAGIC_VALUE_V2) { throw new UnsupportedVersionException("Attempting to use idempotence with a broker which does not " + "support the required message format (v2). The broker must be version 0.11 or later."); } boolean isTransactional = false; - if (transactionState != null) - isTransactional = transactionState.isInTransaction(); + if (transactionManager != null) + isTransactional = transactionManager.isInTransaction(); return MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L, isTransactional); } @@ -440,9 +439,9 @@ public Map> drain(Cluster cluster, // request break; } else { - TransactionState.PidAndEpoch pidAndEpoch = null; - if (transactionState != null) { - pidAndEpoch = transactionState.pidAndEpoch(); + TransactionManager.PidAndEpoch pidAndEpoch = null; + if (transactionManager != null) { + pidAndEpoch = transactionManager.pidAndEpoch(); if (!pidAndEpoch.isValid()) // we cannot send the batch until we have refreshed the PID break; @@ -455,7 +454,7 @@ public Map> drain(Cluster cluster, // the previous attempt may actually have been accepted, and if we change // the pid and sequence here, this attempt will also be accepted, causing // a duplicate. - int sequenceNumber = transactionState.sequenceNumber(batch.topicPartition); + int sequenceNumber = transactionManager.sequenceNumber(batch.topicPartition); log.debug("Dest: {} : producerId: {}, epoch: {}, Assigning sequence for {}: {}", node, pidAndEpoch.producerId, pidAndEpoch.epoch, batch.topicPartition, sequenceNumber); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 68820ee2a8248..698f4de75effb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -22,7 +22,6 @@ import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.RequestCompletionHandler; -import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.MetricName; @@ -110,7 +109,7 @@ public class Sender implements Runnable { private final ApiVersions apiVersions; /* all the state related to transactions, in particular the PID, epoch, and sequence numbers */ - private final TransactionState transactionState; + private final TransactionManager transactionManager; public Sender(KafkaClient client, Metadata metadata, @@ -123,7 +122,7 @@ public Sender(KafkaClient client, Time time, int requestTimeout, long retryBackoffMs, - TransactionState transactionState, + TransactionManager transactionManager, ApiVersions apiVersions) { this.client = client; this.accumulator = accumulator; @@ -138,7 +137,7 @@ public Sender(KafkaClient client, this.requestTimeout = requestTimeout; this.retryBackoffMs = retryBackoffMs; this.apiVersions = apiVersions; - this.transactionState = transactionState; + this.transactionManager = transactionManager; } /** @@ -242,14 +241,14 @@ private long sendProducerData(long now) { // for expired batches. see the documentation of @TransactionState.resetProducerId to understand why // we need to reset the producer id here. for (ProducerBatch expiredBatch : expiredBatches) { - if (transactionState != null && expiredBatch.inRetry()) { + if (transactionManager != null && expiredBatch.inRetry()) { needsTransactionStateReset = true; } this.sensors.recordErrors(expiredBatch.topicPartition.topic(), expiredBatch.recordCount); } if (needsTransactionStateReset) { - transactionState.resetProducerId(); + transactionManager.resetProducerId(); return 0; } @@ -275,22 +274,22 @@ private long sendProducerData(long now) { } private boolean maybeSendTransactionalRequest(long now) { - if (transactionState != null && transactionState.hasInflightTransactionalRequest()) + if (transactionManager != null && transactionManager.hasInflightTransactionalRequest()) return true; - if (transactionState == null || !transactionState.hasPendingTransactionalRequests()) + if (transactionManager == null || !transactionManager.hasPendingTransactionalRequests()) return false; - TransactionState.TransactionalRequest nextRequest = transactionState.nextTransactionalRequest(); + TransactionManager.TransactionalRequest nextRequest = transactionManager.nextTransactionalRequest(); - if (nextRequest.isEndTxnRequest() && transactionState.isCompletingTransaction() && accumulator.hasUnflushedBatches()) { + if (nextRequest.isEndTxnRequest() && transactionManager.isCompletingTransaction() && accumulator.hasUnflushedBatches()) { if (!accumulator.flushInProgress()) accumulator.beginFlush(); - transactionState.reenqueue(nextRequest); + transactionManager.reenqueue(nextRequest); return false; } - if (nextRequest.isEndTxnRequest() && transactionState.isInErrorState()) { + if (nextRequest.isEndTxnRequest() && transactionManager.isInErrorState()) { nextRequest.maybeTerminateWithError(new KafkaException("Cannot commit transaction when there are " + "request errors. Please check your logs for the details of the errors encountered.")); return false; @@ -301,13 +300,13 @@ private boolean maybeSendTransactionalRequest(long now) { while (targetNode == null) { try { if (nextRequest.needsCoordinator()) { - targetNode = transactionState.coordinator(nextRequest.coordinatorType()); + targetNode = transactionManager.coordinator(nextRequest.coordinatorType()); if (targetNode == null) { - transactionState.needsCoordinator(nextRequest); + transactionManager.needsCoordinator(nextRequest); break; } if (!NetworkClientUtils.awaitReady(client, targetNode, time, requestTimeout)) { - transactionState.needsCoordinator(nextRequest); + transactionManager.needsCoordinator(nextRequest); targetNode = null; break; } @@ -320,7 +319,7 @@ private boolean maybeSendTransactionalRequest(long now) { } ClientRequest clientRequest = client.newClientRequest(targetNode.idString(), nextRequest.requestBuilder(), now, true, nextRequest.responseHandler()); - transactionState.setInFlightRequestCorrelationId(clientRequest.correlationId()); + transactionManager.setInFlightRequestCorrelationId(clientRequest.correlationId()); client.send(clientRequest, now); return true; } @@ -332,7 +331,7 @@ private boolean maybeSendTransactionalRequest(long now) { } if (targetNode == null) - transactionState.needsRetry(nextRequest); + transactionManager.needsRetry(nextRequest); return true; } @@ -374,17 +373,17 @@ private Node awaitLeastLoadedNodeReady(long remainingTimeMs) throws IOException private void maybeWaitForPid() { // If this is a transactional producer, the PID will be received when recovering transactions in the // initTransactions() method of the producer. - if (transactionState == null || transactionState.isTransactional()) + if (transactionManager == null || transactionManager.isTransactional()) return; - while (!transactionState.hasPid()) { + while (!transactionManager.hasPid()) { try { Node node = awaitLeastLoadedNodeReady(requestTimeout); if (node != null) { ClientResponse response = sendAndAwaitInitPidRequest(node); if (response.hasResponse() && (response.responseBody() instanceof InitPidResponse)) { InitPidResponse initPidResponse = (InitPidResponse) response.responseBody(); - transactionState.setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch()); + transactionManager.setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch()); } else { log.error("Received an unexpected response type for an InitPidRequest from {}. " + "We will back off and try again.", node); @@ -456,17 +455,17 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons batch.topicPartition, this.retries - batch.attempts() - 1, error); - if (transactionState == null) { + if (transactionManager == null) { reenqueueBatch(batch, now); - } else if (transactionState.pidAndEpoch().producerId == batch.producerId() && transactionState.pidAndEpoch().epoch == batch.producerEpoch()) { + } else if (transactionManager.pidAndEpoch().producerId == batch.producerId() && transactionManager.pidAndEpoch().epoch == batch.producerEpoch()) { // If idempotence is enabled only retry the request if the current PID is the same as the pid of the batch. log.debug("Retrying batch to topic-partition {}. Sequence number : {}", batch.topicPartition, - transactionState.sequenceNumber(batch.topicPartition)); + transactionManager.sequenceNumber(batch.topicPartition)); reenqueueBatch(batch, now); } else { failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " + "batch but the producer id changed from " + batch.producerId() + " to " + - transactionState.pidAndEpoch().producerId + " in the mean time. This batch will be dropped.")); + transactionManager.pidAndEpoch().producerId + " in the mean time. This batch will be dropped.")); this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); } } else { @@ -475,7 +474,7 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons exception = new TopicAuthorizationException(batch.topicPartition.topic()); else exception = error.exception(); - if (error == Errors.OUT_OF_ORDER_SEQUENCE_NUMBER && batch.producerId() == transactionState.pidAndEpoch().producerId) + if (error == Errors.OUT_OF_ORDER_SEQUENCE_NUMBER && batch.producerId() == transactionManager.pidAndEpoch().producerId) log.error("The broker received an out of order sequence number for correlation id {}, topic-partition " + "{} at offset {}. This indicates data loss on the broker, and should be investigated.", correlationId, batch.topicPartition, response.baseOffset); @@ -493,11 +492,11 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons } else { completeBatch(batch, response); - if (transactionState != null && transactionState.pidAndEpoch().producerId == batch.producerId() - && transactionState.pidAndEpoch().epoch == batch.producerEpoch()) { - transactionState.incrementSequenceNumber(batch.topicPartition, batch.recordCount); + if (transactionManager != null && transactionManager.pidAndEpoch().producerId == batch.producerId() + && transactionManager.pidAndEpoch().epoch == batch.producerEpoch()) { + transactionManager.incrementSequenceNumber(batch.topicPartition, batch.recordCount); log.debug("Incremented sequence number for topic-partition {} to {}", batch.topicPartition, - transactionState.sequenceNumber(batch.topicPartition)); + transactionManager.sequenceNumber(batch.topicPartition)); } } @@ -517,12 +516,12 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons } private void failBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, RuntimeException exception) { - if (transactionState != null && !transactionState.isTransactional() - && batch.producerId() == transactionState.pidAndEpoch().producerId) { + if (transactionManager != null && !transactionManager.isTransactional() + && batch.producerId() == transactionManager.pidAndEpoch().producerId) { // Reset the transaction state since we have hit an irrecoverable exception and cannot make any guarantees // about the previously committed message. Note that this will discard the producer id and sequence // numbers for all existing partitions. - transactionState.resetProducerId(); + transactionManager.resetProducerId(); } batch.done(response.baseOffset, response.logAppendTime, exception); this.accumulator.deallocate(batch); @@ -578,8 +577,8 @@ private void sendProduceRequest(long now, int destination, short acks, int timeo } String transactionalId = null; - if (transactionState != null && transactionState.isTransactional()) { - transactionalId = transactionState.transactionalId(); + if (transactionManager != null && transactionManager.isTransactional()) { + transactionalId = transactionManager.transactionalId(); } ProduceRequest.Builder requestBuilder = new ProduceRequest.Builder(minUsedMagic, acks, timeout, produceRecordsByPartition, transactionalId); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java similarity index 98% rename from clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java rename to clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 0224df81ced2d..c89139f2fb782 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.clients.producer; +package org.apache.kafka.clients.producer.internals; import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.RequestCompletionHandler; import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.producer.internals.FutureTransactionalResult; -import org.apache.kafka.clients.producer.internals.TransactionalRequestResult; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; @@ -59,8 +57,8 @@ /** * A class which maintains state for transactions. Also keeps the state necessary to ensure idempotent production. */ -public class TransactionState { - private static final Logger log = LoggerFactory.getLogger(TransactionState.class); +public class TransactionManager { + private static final Logger log = LoggerFactory.getLogger(TransactionManager.class); private static final int NO_INFLIGHT_REQUEST_CORRELATION_ID = -1; @@ -111,7 +109,7 @@ private boolean isTransitionValid(State source, State target) { } } - public TransactionState(String transactionalId, int transactionTimeoutMs) { + public TransactionManager(String transactionalId, int transactionTimeoutMs) { pidAndEpoch = new PidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); sequenceNumbers = new HashMap<>(); this.transactionalId = transactionalId; @@ -130,7 +128,7 @@ public int compare(TransactionalRequest o1, TransactionalRequest o2) { this.pendingTxnOffsetCommits = new HashMap<>(); } - public TransactionState() { + public TransactionManager() { this("", 0); } @@ -516,17 +514,17 @@ private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, Strin } // visible for testing - public boolean transactionContainsPartition(TopicPartition topicPartition) { + boolean transactionContainsPartition(TopicPartition topicPartition) { return isInTransaction() && partitionsInTransaction.contains(topicPartition); } // visible for testing - public boolean hasPendingOffsetCommits() { + boolean hasPendingOffsetCommits() { return isInTransaction() && !pendingTxnOffsetCommits.isEmpty(); } // visible for testing - public boolean isReadyForTransaction() { + boolean isReadyForTransaction() { return isTransactional() && currentState == State.READY; } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java index 7d443b9b70b72..0f1cb54954693 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java @@ -20,7 +20,6 @@ import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; @@ -549,7 +548,7 @@ public void testIdempotenceWithOldMagic() throws InterruptedException { apiVersions.update("foobar", NodeApiVersions.create(Arrays.asList(new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE.id, (short) 0, (short) 2)))); RecordAccumulator accum = new RecordAccumulator(batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * batchSize, - CompressionType.NONE, 10, 100L, metrics, time, apiVersions, new TransactionState()); + CompressionType.NONE, 10, 100L, metrics, time, apiVersions, new TransactionManager()); accum.append(tp1, 0L, key, value, null, 0); } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index f0419914d7019..7de378da2d1f5 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -21,7 +21,6 @@ import org.apache.kafka.clients.MockClient; import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.Node; @@ -377,8 +376,8 @@ public void testMetadataTopicExpiry() throws Exception { @Test public void testInitPidRequest() throws Exception { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(); - setupWithTransactionState(transactionState); + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); client.setNode(new Node(1, "localhost", 33343)); client.prepareResponse(new MockClient.RequestMatcher() { @Override @@ -387,17 +386,17 @@ public boolean matches(AbstractRequest body) { } }, new InitPidResponse(Errors.NONE, producerId, (short) 0)); sender.run(time.milliseconds()); - assertTrue(transactionState.hasPid()); - assertEquals(producerId, transactionState.pidAndEpoch().producerId); - assertEquals((short) 0, transactionState.pidAndEpoch().epoch); + assertTrue(transactionManager.hasPid()); + assertEquals(producerId, transactionManager.pidAndEpoch().producerId); + assertEquals((short) 0, transactionManager.pidAndEpoch().epoch); } @Test public void testSequenceNumberIncrement() throws InterruptedException { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(); - transactionState.setPidAndEpoch(producerId, (short) 0); - setupWithTransactionState(transactionState); + TransactionManager transactionManager = new TransactionManager(); + transactionManager.setPidAndEpoch(producerId, (short) 0); + setupWithTransactionState(transactionManager); client.setNode(new Node(1, "localhost", 33343)); int maxRetries = 10; @@ -413,7 +412,7 @@ public void testSequenceNumberIncrement() throws InterruptedException { time, REQUEST_TIMEOUT, 50, - transactionState, + transactionManager, apiVersions ); @@ -442,15 +441,15 @@ public boolean matches(AbstractRequest body) { sender.run(time.milliseconds()); // receive response assertTrue(responseFuture.isDone()); - assertEquals((long) transactionState.sequenceNumber(tp0), 1L); + assertEquals((long) transactionManager.sequenceNumber(tp0), 1L); } @Test public void testAbortRetryWhenPidChanges() throws InterruptedException { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(); - transactionState.setPidAndEpoch(producerId, (short) 0); - setupWithTransactionState(transactionState); + TransactionManager transactionManager = new TransactionManager(); + transactionManager.setPidAndEpoch(producerId, (short) 0); + setupWithTransactionState(transactionManager); client.setNode(new Node(1, "localhost", 33343)); int maxRetries = 10; @@ -466,7 +465,7 @@ public void testAbortRetryWhenPidChanges() throws InterruptedException { time, REQUEST_TIMEOUT, 50, - transactionState, + transactionManager, apiVersions ); @@ -481,7 +480,7 @@ public void testAbortRetryWhenPidChanges() throws InterruptedException { assertEquals(0, client.inFlightRequestCount()); assertFalse("Client ready status should be false", client.isReady(node, 0L)); - transactionState.setPidAndEpoch(producerId + 1, (short) 0); + transactionManager.setPidAndEpoch(producerId + 1, (short) 0); sender.run(time.milliseconds()); // receive error sender.run(time.milliseconds()); // reconnect sender.run(time.milliseconds()); // nothing to do, since the pid has changed. We should check the metrics for errors. @@ -491,15 +490,15 @@ public void testAbortRetryWhenPidChanges() throws InterruptedException { assertTrue("Expected non-zero value for record send errors", recordErrors.value() > 0); assertTrue(responseFuture.isDone()); - assertEquals((long) transactionState.sequenceNumber(tp0), 0L); + assertEquals((long) transactionManager.sequenceNumber(tp0), 0L); } @Test public void testResetWhenOutOfOrderSequenceReceived() throws InterruptedException { final long producerId = 343434L; - TransactionState transactionState = new TransactionState(); - transactionState.setPidAndEpoch(producerId, (short) 0); - setupWithTransactionState(transactionState); + TransactionManager transactionManager = new TransactionManager(); + transactionManager.setPidAndEpoch(producerId, (short) 0); + setupWithTransactionState(transactionManager); client.setNode(new Node(1, "localhost", 33343)); int maxRetries = 10; @@ -515,7 +514,7 @@ public void testResetWhenOutOfOrderSequenceReceived() throws InterruptedExceptio time, REQUEST_TIMEOUT, 50, - transactionState, + transactionManager, apiVersions ); @@ -529,7 +528,7 @@ public void testResetWhenOutOfOrderSequenceReceived() throws InterruptedExceptio sender.run(time.milliseconds()); assertTrue(responseFuture.isDone()); - assertFalse("Expected transaction state to be reset upon receiving an OutOfOrderSequenceException", transactionState.hasPid()); + assertFalse("Expected transaction state to be reset upon receiving an OutOfOrderSequenceException", transactionManager.hasPid()); } private void completedWithError(Future future, Errors error) throws Exception { @@ -548,12 +547,12 @@ private ProduceResponse produceResponse(TopicPartition tp, long offset, Errors e return new ProduceResponse(partResp, throttleTimeMs); } - private void setupWithTransactionState(TransactionState transactionState) { + private void setupWithTransactionState(TransactionManager transactionManager) { Map metricTags = new LinkedHashMap<>(); metricTags.put("client-id", CLIENT_ID); MetricConfig metricConfig = new MetricConfig().tags(metricTags); this.metrics = new Metrics(metricConfig, time); - this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionState); + this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionManager); this.sender = new Sender(this.client, this.metadata, this.accumulator, @@ -565,7 +564,7 @@ private void setupWithTransactionState(TransactionState transactionState) { this.time, REQUEST_TIMEOUT, 50, - transactionState, + transactionManager, apiVersions); this.metadata.update(this.cluster, Collections.emptySet(), time.milliseconds()); } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java similarity index 78% rename from clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java rename to clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index f8ecc1d82c6a3..3a72d0d57b97a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.clients.producer.TransactionState; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; @@ -71,7 +70,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class TransactionsTest { +public class TransactionManagerTest { private static final int MAX_REQUEST_SIZE = 1024 * 1024; private static final short ACKS_ALL = -1; private static final int MAX_RETRIES = 0; @@ -91,7 +90,7 @@ public class TransactionsTest { private Cluster cluster = TestUtils.singletonCluster("test", 2); private RecordAccumulator accumulator = null; private Sender sender = null; - private TransactionState transactionState = null; + private TransactionManager transactionManager = null; private Node brokerNode = null; @Before @@ -101,9 +100,9 @@ public void setup() { int batchSize = 16 * 1024; MetricConfig metricConfig = new MetricConfig().tags(metricTags); this.brokerNode = new Node(0, "localhost", 2211); - this.transactionState = new TransactionState(transactionalId, transactionTimeoutMs); + this.transactionManager = new TransactionManager(transactionalId, transactionTimeoutMs); Metrics metrics = new Metrics(metricConfig, time); - this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionState); + this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionManager); this.sender = new Sender(this.client, this.metadata, this.accumulator, @@ -115,11 +114,35 @@ public void setup() { this.time, REQUEST_TIMEOUT, 50, - transactionState, + transactionManager, apiVersions); this.metadata.update(this.cluster, Collections.emptySet(), time.milliseconds()); } + @Test(expected = IllegalStateException.class) + public void testInvalidSequenceIncrement() { + TransactionManager transactionManager = new TransactionManager(); + transactionManager.incrementSequenceNumber(tp0, 3333); + } + + @Test + public void testDefaultSequenceNumber() { + TransactionManager transactionManager = new TransactionManager(); + assertEquals((int) transactionManager.sequenceNumber(tp0), 0); + transactionManager.incrementSequenceNumber(tp0, 3); + assertEquals((int) transactionManager.sequenceNumber(tp0), 3); + } + + + @Test + public void testProducerIdReset() { + TransactionManager transactionManager = new TransactionManager(); + assertEquals((int) transactionManager.sequenceNumber(tp0), 0); + transactionManager.incrementSequenceNumber(tp0, 3); + assertEquals((int) transactionManager.sequenceNumber(tp0), 3); + transactionManager.resetProducerId(); + assertEquals((int) transactionManager.sequenceNumber(tp0), 0); + } @Test public void testBasicTransaction() throws InterruptedException { @@ -128,19 +151,19 @@ public void testBasicTransaction() throws InterruptedException { // It finds the coordinator and then gets a PID. final long pid = 13131L; final short epoch = 1; - transactionState.initializeTransactions(); + transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.NONE, false, pid, epoch); sender.run(time.milliseconds()); // get pid. - assertTrue(transactionState.hasPid()); - transactionState.beginTransaction(); - transactionState.maybeAddPartitionToTransaction(tp0); + assertTrue(transactionManager.hasPid()); + transactionManager.beginTransaction(); + transactionManager.maybeAddPartitionToTransaction(tp0); Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; @@ -149,10 +172,10 @@ public void testBasicTransaction() throws InterruptedException { prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); prepareProduceResponse(Errors.NONE, pid, epoch); - assertFalse(transactionState.transactionContainsPartition(tp0)); + assertFalse(transactionManager.transactionContainsPartition(tp0)); sender.run(time.milliseconds()); // send addPartitions. // Check that only addPartitions was sent. - assertTrue(transactionState.transactionContainsPartition(tp0)); + assertTrue(transactionManager.transactionContainsPartition(tp0)); assertFalse(responseFuture.isDone()); sender.run(time.milliseconds()); // send produce request. @@ -161,9 +184,9 @@ public void testBasicTransaction() throws InterruptedException { Map offsets = new HashMap<>(); offsets.put(tp1, new OffsetAndMetadata(1)); final String consumerGroupId = "myconsumergroup"; - FutureTransactionalResult addOffsetsResult = transactionState.sendOffsetsToTransaction(offsets, consumerGroupId); + FutureTransactionalResult addOffsetsResult = transactionManager.sendOffsetsToTransaction(offsets, consumerGroupId); - assertFalse(transactionState.hasPendingOffsetCommits()); + assertFalse(transactionManager.hasPendingOffsetCommits()); client.prepareResponse(new MockClient.RequestMatcher() { @Override @@ -178,7 +201,7 @@ public boolean matches(AbstractRequest body) { }, new AddOffsetsToTxnResponse(Errors.NONE)); sender.run(time.milliseconds()); // Send AddOffsetsRequest - assertTrue(transactionState.hasPendingOffsetCommits()); // We should now have created and queued the offset commit request. + assertTrue(transactionManager.hasPendingOffsetCommits()); // We should now have created and queued the offset commit request. assertFalse(addOffsetsResult.isDone()); Map txnOffsetCommitResponse = new HashMap<>(); @@ -197,24 +220,24 @@ public boolean matches(AbstractRequest body) { } }, new TxnOffsetCommitResponse(txnOffsetCommitResponse)); - assertEquals(null, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP)); + assertEquals(null, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP)); sender.run(time.milliseconds()); // try to send TxnOffsetCommitRequest, but find we don't have a group coordinator. sender.run(time.milliseconds()); // send find coordinator for group request - assertNotNull(transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP)); - assertTrue(transactionState.hasPendingOffsetCommits()); + assertNotNull(transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP)); + assertTrue(transactionManager.hasPendingOffsetCommits()); sender.run(time.milliseconds()); // send TxnOffsetCommitRequest commit. - assertFalse(transactionState.hasPendingOffsetCommits()); + assertFalse(transactionManager.hasPendingOffsetCommits()); assertTrue(addOffsetsResult.isDone()); // We should only be done after both RPCs complete. - transactionState.beginCommittingTransaction(); + transactionManager.beginCommittingTransaction(); prepareEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch); sender.run(time.milliseconds()); // commit. - assertFalse(transactionState.isInTransaction()); - assertFalse(transactionState.isCompletingTransaction()); - assertFalse(transactionState.transactionContainsPartition(tp0)); + assertFalse(transactionManager.isInTransaction()); + assertFalse(transactionManager.isCompletingTransaction()); + assertFalse(transactionManager.transactionContainsPartition(tp0)); } @Test @@ -222,13 +245,13 @@ public void testDisconnectAndRetry() { client.setNode(brokerNode); // This is called from the initTransactions method in the producer as the first order of business. // It finds the coordinator and then gets a PID. - transactionState.initializeTransactions(); + transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, true, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator, connection lost. prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); } @Test @@ -238,29 +261,29 @@ public void testCoordinatorLost() { // It finds the coordinator and then gets a PID. final long pid = 13131L; final short epoch = 1; - FutureTransactionalResult initPidResult = transactionState.initializeTransactions(); + FutureTransactionalResult initPidResult = transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.NOT_COORDINATOR, false, pid, epoch); sender.run(time.milliseconds()); // send pid, get not coordinator. Should resend the FindCoordinator and InitPid requests - assertEquals(null, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(null, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); assertFalse(initPidResult.isDone()); - assertFalse(transactionState.hasPid()); + assertFalse(transactionManager.hasPid()); prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); assertFalse(initPidResult.isDone()); prepareInitPidResponse(Errors.NONE, false, pid, epoch); sender.run(time.milliseconds()); // get pid and epoch assertTrue(initPidResult.isDone()); // The future should only return after the second round of retries succeed. - assertTrue(transactionState.hasPid()); - assertEquals(pid, transactionState.pidAndEpoch().producerId); - assertEquals(epoch, transactionState.pidAndEpoch().epoch); + assertTrue(transactionManager.hasPid()); + assertEquals(pid, transactionManager.pidAndEpoch().producerId); + assertEquals(epoch, transactionManager.pidAndEpoch().epoch); } @Test @@ -270,38 +293,38 @@ public void testFlushPendingPartitionsOnCommit() throws InterruptedException { // It finds the coordinator and then gets a PID. final long pid = 13131L; final short epoch = 1; - transactionState.initializeTransactions(); + transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.NONE, false, pid, epoch); sender.run(time.milliseconds()); // get pid. - assertTrue(transactionState.hasPid()); + assertTrue(transactionManager.hasPid()); - transactionState.beginTransaction(); - transactionState.maybeAddPartitionToTransaction(tp0); + transactionManager.beginTransaction(); + transactionManager.maybeAddPartitionToTransaction(tp0); Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; assertFalse(responseFuture.isDone()); - FutureTransactionalResult commitResult = transactionState.beginCommittingTransaction(); + FutureTransactionalResult commitResult = transactionManager.beginCommittingTransaction(); // we have an append, an add partitions request, and now also an endtxn. // The order should be: // 1. Add Partitions // 2. Produce // 3. EndTxn. - assertFalse(transactionState.transactionContainsPartition(tp0)); + assertFalse(transactionManager.transactionContainsPartition(tp0)); prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); sender.run(time.milliseconds()); // AddPartitions. - assertTrue(transactionState.transactionContainsPartition(tp0)); + assertTrue(transactionManager.transactionContainsPartition(tp0)); assertFalse(responseFuture.isDone()); assertFalse(commitResult.isDone()); @@ -311,12 +334,12 @@ public void testFlushPendingPartitionsOnCommit() throws InterruptedException { prepareEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch); assertFalse(commitResult.isDone()); - assertTrue(transactionState.isInTransaction()); - assertTrue(transactionState.isCompletingTransaction()); + assertTrue(transactionManager.isInTransaction()); + assertTrue(transactionManager.isCompletingTransaction()); sender.run(time.milliseconds()); assertTrue(commitResult.isDone()); - assertFalse(transactionState.isInTransaction()); + assertFalse(transactionManager.isInTransaction()); } @Test @@ -326,20 +349,20 @@ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedExcept // It finds the coordinator and then gets a PID. final long pid = 13131L; final short epoch = 1; - transactionState.initializeTransactions(); + transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.NONE, false, pid, epoch); sender.run(time.milliseconds()); // get pid. - assertTrue(transactionState.hasPid()); - transactionState.beginTransaction(); + assertTrue(transactionManager.hasPid()); + transactionManager.beginTransaction(); // User does one producer.sed - transactionState.maybeAddPartitionToTransaction(tp0); + transactionManager.maybeAddPartitionToTransaction(tp0); Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; @@ -347,29 +370,29 @@ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedExcept assertFalse(responseFuture.isDone()); prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); - assertFalse(transactionState.transactionContainsPartition(tp0)); + assertFalse(transactionManager.transactionContainsPartition(tp0)); // Sender flushes one add partitions. The produce goes next. sender.run(time.milliseconds()); // send addPartitions. // Check that only addPartitions was sent. - assertTrue(transactionState.transactionContainsPartition(tp0)); + assertTrue(transactionManager.transactionContainsPartition(tp0)); // In the mean time, the user does a second produce to a different partition - transactionState.maybeAddPartitionToTransaction(tp1); + transactionManager.maybeAddPartitionToTransaction(tp1); Future secondResponseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; prepareAddPartitionsToTxnResponse(Errors.NONE, tp1, epoch, pid); prepareProduceResponse(Errors.NONE, pid, epoch); - assertFalse(transactionState.transactionContainsPartition(tp1)); + assertFalse(transactionManager.transactionContainsPartition(tp1)); assertFalse(responseFuture.isDone()); assertFalse(secondResponseFuture.isDone()); // The second add partitionsh should go out here. sender.run(time.milliseconds()); // send second add partitions request - assertTrue(transactionState.transactionContainsPartition(tp1)); + assertTrue(transactionManager.transactionContainsPartition(tp1)); assertFalse(responseFuture.isDone()); assertFalse(secondResponseFuture.isDone()); @@ -388,19 +411,19 @@ public void testProducerFencedException() throws InterruptedException, Execution // It finds the coordinator and then gets a PID. final long pid = 13131L; final short epoch = 1; - transactionState.initializeTransactions(); + transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.NONE, false, pid, epoch); sender.run(time.milliseconds()); // get pid. - assertTrue(transactionState.hasPid()); - transactionState.beginTransaction(); - transactionState.maybeAddPartitionToTransaction(tp0); + assertTrue(transactionManager.hasPid()); + transactionManager.beginTransaction(); + transactionManager.maybeAddPartitionToTransaction(tp0); Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; @@ -422,24 +445,24 @@ public void testDisallowCommitOnProduceFailure() throws InterruptedException { // It finds the coordinator and then gets a PID. final long pid = 13131L; final short epoch = 1; - transactionState.initializeTransactions(); + transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator - assertEquals(brokerNode, transactionState.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); + assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.NONE, false, pid, epoch); sender.run(time.milliseconds()); // get pid. - assertTrue(transactionState.hasPid()); - transactionState.beginTransaction(); - transactionState.maybeAddPartitionToTransaction(tp0); + assertTrue(transactionManager.hasPid()); + transactionManager.beginTransaction(); + transactionManager.maybeAddPartitionToTransaction(tp0); Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), - "value".getBytes(), new MockCallback(transactionState), MAX_BLOCK_TIMEOUT).future; + "value".getBytes(), new MockCallback(transactionManager), MAX_BLOCK_TIMEOUT).future; - FutureTransactionalResult commitResult = transactionState.beginCommittingTransaction(); + FutureTransactionalResult commitResult = transactionManager.beginCommittingTransaction(); assertFalse(responseFuture.isDone()); prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); prepareProduceResponse(Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, pid, epoch); @@ -459,24 +482,24 @@ public void testDisallowCommitOnProduceFailure() throws InterruptedException { } // Commit is not allowed, so let's abort and try again. - FutureTransactionalResult abortResult = transactionState.beginAbortingTransaction(); + FutureTransactionalResult abortResult = transactionManager.beginAbortingTransaction(); prepareEndTxnResponse(Errors.NONE, TransactionResult.ABORT, pid, epoch); sender.run(time.milliseconds()); // Send abort request. It is valid to transition from ERROR to ABORT assertTrue(abortResult.isDone()); assertTrue(abortResult.get().isSuccessful()); - assertTrue(transactionState.isReadyForTransaction()); // make sure we are ready for a transaction now. + assertTrue(transactionManager.isReadyForTransaction()); // make sure we are ready for a transaction now. } private static class MockCallback implements Callback { - private final TransactionState transactionState; - public MockCallback(TransactionState transactionState) { - this.transactionState = transactionState; + private final TransactionManager transactionManager; + public MockCallback(TransactionManager transactionManager) { + this.transactionManager = transactionManager; } @Override public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null && transactionState != null) { - transactionState.maybeSetError(exception); + if (exception != null && transactionManager != null) { + transactionManager.maybeSetError(exception); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java deleted file mode 100644 index 2b0e7275e6e7b..0000000000000 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.producer.internals; - - -import org.apache.kafka.clients.producer.TransactionState; -import org.apache.kafka.common.TopicPartition; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class TransactionStateTest { - - private TopicPartition topicPartition; - - @Before - public void setUp() { - topicPartition = new TopicPartition("topic-0", 0); - } - - @Test(expected = IllegalStateException.class) - public void testInvalidSequenceIncrement() { - TransactionState transactionState = new TransactionState(); - transactionState.incrementSequenceNumber(topicPartition, 3333); - } - - @Test - public void testDefaultSequenceNumber() { - TransactionState transactionState = new TransactionState(); - assertEquals((int) transactionState.sequenceNumber(topicPartition), 0); - transactionState.incrementSequenceNumber(topicPartition, 3); - assertEquals((int) transactionState.sequenceNumber(topicPartition), 3); - } - - - @Test - public void testProducerIdReset() { - TransactionState transactionState = new TransactionState(); - assertEquals((int) transactionState.sequenceNumber(topicPartition), 0); - transactionState.incrementSequenceNumber(topicPartition, 3); - assertEquals((int) transactionState.sequenceNumber(topicPartition), 3); - transactionState.resetProducerId(); - assertEquals((int) transactionState.sequenceNumber(topicPartition), 0); - } -} From c8ccfde59e7fb97a0f787607fba7b224ee36fc66 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 26 Apr 2017 15:28:49 -0700 Subject: [PATCH 15/21] Throw an exception when a state transition fails detailing the failure. We were previously logging the details and throwing a generic exception, which was not very helpful. --- .../internals/TransactionManager.java | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index c89139f2fb782..adb4e119a82c8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -232,9 +232,7 @@ public boolean isValid() { public synchronized FutureTransactionalResult initializeTransactions() { ensureTransactional(); - if (!transitionTo(State.INITIALIZING)) - throw new KafkaException("Could not initialize transactions. Either transactions have already been " + - "initialized or are being initialized."); + transitionTo(State.INITIALIZING); setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); this.sequenceNumbers.clear(); if (transactionCoordinator == null) @@ -253,16 +251,13 @@ public synchronized FutureTransactionalResult initializeTransactions() { public synchronized void beginTransaction() { ensureTransactional(); maybeFailWithError(); - if (!transitionTo(State.IN_TRANSACTION)) - throw new KafkaException("Producer isn't ready to begin a transaction, most likely because there is " + - "already an ongoing transaction."); + transitionTo(State.IN_TRANSACTION); } public synchronized FutureTransactionalResult beginCommittingTransaction() { ensureTransactional(); maybeFailWithError(); - if (!transitionTo(State.COMMITTING_TRANSACTION)) - throw new KafkaException("Cannot commit transaction, most likely because a transaction is already being completed."); + transitionTo(State.COMMITTING_TRANSACTION); return beginCompletingTransaction(true); } @@ -270,9 +265,7 @@ public synchronized FutureTransactionalResult beginAbortingTransaction() { ensureTransactional(); if (isFenced()) throw new ProducerFencedException("There is a newer producer using the same transactional.id."); - if (!transitionTo(State.ABORTING_TRANSACTION)) - throw new KafkaException("Cannot abort transaction, either because a transaction is already " + - "being completed at the moment, or because there has been an error with a previous request."); + transitionTo(State.ABORTING_TRANSACTION); return beginCompletingTransaction(false); } @@ -293,7 +286,7 @@ public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map Date: Wed, 26 Apr 2017 16:01:37 -0700 Subject: [PATCH 16/21] Fix build errors aftre rebase, miscellaneous cleanups --- .../main/java/org/apache/kafka/common/record/MemoryRecords.java | 2 +- .../clients/producer/internals/TransactionManagerTest.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java index 16588c2302863..548cd451007e9 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java @@ -177,7 +177,7 @@ private static FilterResult filterTo(Iterable batches, Recor long logAppendTime = timestampType == TimestampType.LOG_APPEND_TIME ? batch.maxTimestamp() : RecordBatch.NO_TIMESTAMP; MemoryRecordsBuilder builder = builder(slice, batch.magic(), batch.compressionType(), timestampType, firstOffset, logAppendTime, batch.producerId(), batch.producerEpoch(), batch.baseSequence(), - batch.partitionLeaderEpoch()); + batch.isTransactional(), batch.partitionLeaderEpoch()); for (Record record : retainedRecords) builder.append(record); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index 3a72d0d57b97a..7c5b2b57fcf49 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -501,7 +501,6 @@ public void onCompletion(RecordMetadata metadata, Exception exception) { if (exception != null && transactionManager != null) { transactionManager.maybeSetError(exception); } - } } From 34c7e36180514acd30ab10a9e33fe501ee5fc80b Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 26 Apr 2017 16:49:55 -0700 Subject: [PATCH 17/21] Fix the visibility of various methods in the TransactionManager. Made a bunch package-private, since the class is now in the same package as the sender --- .../internals/TransactionManager.java | 159 +++++++++--------- 1 file changed, 81 insertions(+), 78 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index adb4e119a82c8..12d331a517d27 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -132,7 +132,7 @@ public TransactionManager() { this("", 0); } - public static class TransactionalRequest { + static class TransactionalRequest { private enum Priority { FIND_COORDINATOR(0), INIT_PRODUCER_ID(1), @@ -174,31 +174,31 @@ private TransactionalRequest(AbstractRequest.Builder requestBuilder, RequestC this.result = result; } - public AbstractRequest.Builder requestBuilder() { + AbstractRequest.Builder requestBuilder() { return requestBuilder; } - public boolean needsCoordinator() { + boolean needsCoordinator() { return coordinatorType != null; } - public FindCoordinatorRequest.CoordinatorType coordinatorType() { + FindCoordinatorRequest.CoordinatorType coordinatorType() { return coordinatorType; } - public RequestCompletionHandler responseHandler() { + RequestCompletionHandler responseHandler() { return handler; } - public boolean isRetry() { + boolean isRetry() { return isRetry; } - public boolean isEndTxnRequest() { + boolean isEndTxnRequest() { return priority == Priority.END_TXN; } - public boolean maybeTerminateWithError(RuntimeException error) { + boolean maybeTerminateWithError(RuntimeException error) { if (result != null) { result.setError(error); result.done(); @@ -216,7 +216,7 @@ private Priority priority() { } } - public static class PidAndEpoch { + static class PidAndEpoch { public final long producerId; public final short epoch; @@ -300,13 +300,56 @@ public synchronized void maybeAddPartitionToTransaction(TopicPartition topicPart newPartitionsToBeAddedToTransaction.add(topicPartition); } + public Exception lastError() { + return lastError; + } + + public String transactionalId() { + return transactionalId; + } + + public boolean hasPid() { + return pidAndEpoch.isValid(); + } + + public boolean isTransactional() { + return transactionalId != null && !transactionalId.isEmpty(); + } + + public boolean isFenced() { + return currentState == State.FENCED; + } + + public boolean isCompletingTransaction() { + return currentState == State.COMMITTING_TRANSACTION || currentState == State.ABORTING_TRANSACTION; + } + + public boolean isInTransaction() { + return currentState == State.IN_TRANSACTION || isCompletingTransaction(); + } + + public boolean isInErrorState() { + return currentState == State.ERROR; + } + + public synchronized boolean maybeSetError(Exception exception) { + if (isTransactional() && isInTransaction()) { + if (exception instanceof ProducerFencedException) + transitionTo(State.FENCED, exception); + else + transitionTo(State.ERROR, exception); + return true; + } + return false; + } + /** * Get the current pid and epoch without blocking. Callers must use {@link PidAndEpoch#isValid()} to * verify that the result is valid. * * @return the current PidAndEpoch. */ - public PidAndEpoch pidAndEpoch() { + PidAndEpoch pidAndEpoch() { return pidAndEpoch; } @@ -314,7 +357,7 @@ public PidAndEpoch pidAndEpoch() { * Set the pid and epoch atomically. This method will signal any callers blocked on the `pidAndEpoch` method * once the pid is set. This method will be called on the background thread when the broker responds with the pid. */ - public synchronized void setPidAndEpoch(long pid, short epoch) { + synchronized void setPidAndEpoch(long pid, short epoch) { this.pidAndEpoch = new PidAndEpoch(pid, epoch); } @@ -335,7 +378,7 @@ public synchronized void setPidAndEpoch(long pid, short epoch) { * would not have any way of knowing this happened. So for the transactional producer, it's best to return the * produce error to the user and let them abort the transaction and close the producer explicitly. */ - public synchronized void resetProducerId() { + synchronized void resetProducerId() { if (isTransactional()) throw new IllegalStateException("Cannot reset producer state for a transactional producer. " + "You must either abort the ongoing transaction or reinitialize the transactional producer instead"); @@ -346,7 +389,7 @@ public synchronized void resetProducerId() { /** * Returns the next sequence number to be written to the given TopicPartition. */ - public synchronized Integer sequenceNumber(TopicPartition topicPartition) { + synchronized Integer sequenceNumber(TopicPartition topicPartition) { Integer currentSequenceNumber = sequenceNumbers.get(topicPartition); if (currentSequenceNumber == null) { currentSequenceNumber = 0; @@ -355,7 +398,7 @@ public synchronized Integer sequenceNumber(TopicPartition topicPartition) { return currentSequenceNumber; } - public synchronized void incrementSequenceNumber(TopicPartition topicPartition, int increment) { + synchronized void incrementSequenceNumber(TopicPartition topicPartition, int increment) { Integer currentSequenceNumber = sequenceNumbers.get(topicPartition); if (currentSequenceNumber == null) throw new IllegalStateException("Attempt to increment sequence number for a partition with no current sequence."); @@ -364,12 +407,12 @@ public synchronized void incrementSequenceNumber(TopicPartition topicPartition, sequenceNumbers.put(topicPartition, currentSequenceNumber); } - public boolean hasPendingTransactionalRequests() { + boolean hasPendingTransactionalRequests() { return !(pendingTransactionalRequests.isEmpty() && newPartitionsToBeAddedToTransaction.isEmpty()); } - public TransactionalRequest nextTransactionalRequest() { + TransactionalRequest nextTransactionalRequest() { if (!hasPendingTransactionalRequests()) return null; @@ -379,49 +422,18 @@ public TransactionalRequest nextTransactionalRequest() { return pendingTransactionalRequests.poll(); } - public Exception lastError() { - return lastError; - } - - public String transactionalId() { - return transactionalId; - } - public boolean hasPid() { - return pidAndEpoch.isValid(); - } - - public boolean isTransactional() { - return transactionalId != null && !transactionalId.isEmpty(); - } - public boolean isFenced() { - return currentState == State.FENCED; - } - - public boolean isCompletingTransaction() { - return currentState == State.COMMITTING_TRANSACTION || currentState == State.ABORTING_TRANSACTION; - } - - public boolean isInTransaction() { - return currentState == State.IN_TRANSACTION || isCompletingTransaction(); - } - - public boolean isInErrorState() { - return currentState == State.ERROR; - } - - - public void needsRetry(TransactionalRequest request) { + void needsRetry(TransactionalRequest request) { request.setRetry(); pendingTransactionalRequests.add(request); } - public void reenqueue(TransactionalRequest request) { + void reenqueue(TransactionalRequest request) { pendingTransactionalRequests.add(request); } - public Node coordinator(FindCoordinatorRequest.CoordinatorType type) { + Node coordinator(FindCoordinatorRequest.CoordinatorType type) { switch (type) { case GROUP: return consumerGroupCoordinator; @@ -432,33 +444,38 @@ public Node coordinator(FindCoordinatorRequest.CoordinatorType type) { } } - public void needsCoordinator(TransactionalRequest request) { + void needsCoordinator(TransactionalRequest request) { needsCoordinator(request.coordinatorType, request.coordinatorKey); } - public synchronized boolean maybeSetError(Exception exception) { - if (isTransactional() && isInTransaction()) { - if (exception instanceof ProducerFencedException) - transitionTo(State.FENCED, exception); - else - transitionTo(State.ERROR, exception); - return true; - } - return false; - } - public void setInFlightRequestCorrelationId(int correlationId) { + void setInFlightRequestCorrelationId(int correlationId) { inFlightRequestCorrelationId = correlationId; } - public void resetInFlightRequestCorrelationId() { + void resetInFlightRequestCorrelationId() { inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID; } - public boolean hasInflightTransactionalRequest() { + boolean hasInflightTransactionalRequest() { return inFlightRequestCorrelationId != NO_INFLIGHT_REQUEST_CORRELATION_ID; } + // visible for testing + boolean transactionContainsPartition(TopicPartition topicPartition) { + return isInTransaction() && partitionsInTransaction.contains(topicPartition); + } + + // visible for testing + boolean hasPendingOffsetCommits() { + return isInTransaction() && !pendingTxnOffsetCommits.isEmpty(); + } + + // visible for testing + boolean isReadyForTransaction() { + return isTransactional() && currentState == State.READY; + } + private void transitionTo(State target) { transitionTo(target, null); } @@ -504,20 +521,6 @@ private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, Strin pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, false)); } - // visible for testing - boolean transactionContainsPartition(TopicPartition topicPartition) { - return isInTransaction() && partitionsInTransaction.contains(topicPartition); - } - - // visible for testing - boolean hasPendingOffsetCommits() { - return isInTransaction() && !pendingTxnOffsetCommits.isEmpty(); - } - - // visible for testing - boolean isReadyForTransaction() { - return isTransactional() && currentState == State.READY; - } private void completeTransaction() { transitionTo(State.READY); From 63c4596c88436fcb7b21e4e729867d7e3124a918 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 26 Apr 2017 17:11:15 -0700 Subject: [PATCH 18/21] Handle the newly introduced CONCURRENT_TRANSACTIONS error code --- .../producer/internals/TransactionManager.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 12d331a517d27..24f6b2fc86a17 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -422,8 +422,6 @@ TransactionalRequest nextTransactionalRequest() { return pendingTransactionalRequests.poll(); } - - void needsRetry(TransactionalRequest request) { request.setRetry(); pendingTransactionalRequests.add(request); @@ -448,7 +446,6 @@ void needsCoordinator(TransactionalRequest request) { needsCoordinator(request.coordinatorType, request.coordinatorKey); } - void setInFlightRequestCorrelationId(int correlationId) { inFlightRequestCorrelationId = correlationId; } @@ -639,7 +636,7 @@ public void handleResponse(AbstractResponse responseBody) { } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); - } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) { reenqueue(); } else { result.setError(error.exception()); @@ -672,7 +669,7 @@ public void handleResponse(AbstractResponse response) { } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); - } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) { reenqueue(); } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) { log.error("Seems like the broker has bad transaction state. producerId: {}, error: {}. message: {}", @@ -753,7 +750,7 @@ public void handleResponse(AbstractResponse responseBody) { } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); - } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) { reenqueue(); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { transitionTo(State.FENCED, error.exception()); @@ -792,7 +789,7 @@ public void handleResponse(AbstractResponse responseBody) { } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); - } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) { + } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) { reenqueue(); } else if (error == Errors.INVALID_PRODUCER_EPOCH) { transitionTo(State.FENCED, error.exception()); From 05f1ad67425a912adbc320be3775e497890a3c8f Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 26 Apr 2017 17:30:18 -0700 Subject: [PATCH 19/21] Minor improvements to the TransactionalManager --- .../kafka/clients/producer/internals/TransactionManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 24f6b2fc86a17..16dccc50a2d96 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -589,7 +589,8 @@ private abstract class TransactionalRequestCallBack implements RequestCompletion public void onComplete(ClientResponse response) { if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) { log.error("Detected more than one inflight transactional request. This should never happen."); - transitionTo(State.ERROR); + transitionTo(State.ERROR, new RuntimeException("Detected more than one inflight transactional request. This should never happen.")); + return; } resetInFlightRequestCorrelationId(); @@ -851,6 +852,7 @@ public void handleResponse(AbstractResponse responseBody) { return; } + // retry the commits which failed with a retriable error. if (!pendingTxnOffsetCommits.isEmpty()) pendingTransactionalRequests.add(txnOffsetCommitRequest(consumerGroupId, true, result)); From a88ac0858daf083db22f01bef97f56a88c9fd611 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 27 Apr 2017 10:48:03 -0700 Subject: [PATCH 20/21] Add the existing topicPartition object to a transaction. Otherwise we won't add the partition if it isn't explicitly added to the record --- .../java/org/apache/kafka/clients/producer/KafkaProducer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 1045f4c4798a3..d171c29981748 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -667,7 +667,7 @@ private Future doSend(ProducerRecord record, Callback call Callback interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp, transactionManager); if (transactionManager != null && transactionManager.isInTransaction()) { - transactionManager.maybeAddPartitionToTransaction(new TopicPartition(record.topic(), record.partition())); + transactionManager.maybeAddPartitionToTransaction(tp); } From 4cff3f7eea9c5d559f2776e6d5e3be11578fcf02 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 27 Apr 2017 11:32:48 -0700 Subject: [PATCH 21/21] Minor cleanup --- .../java/org/apache/kafka/clients/producer/KafkaProducer.java | 4 +--- .../kafka/clients/producer/internals/TransactionManager.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index d171c29981748..3745aba8b81a5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -666,10 +666,8 @@ private Future doSend(ProducerRecord record, Callback call // producer callback will make sure to call both 'callback' and interceptor callback Callback interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp, transactionManager); - if (transactionManager != null && transactionManager.isInTransaction()) { + if (transactionManager != null) transactionManager.maybeAddPartitionToTransaction(tp); - } - RecordAccumulator.RecordAppendResult result = accumulator.append(tp, timestamp, serializedKey, serializedValue, interceptCallback, remainingWaitMs); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 16dccc50a2d96..644d4f8d5aeef 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -295,7 +295,7 @@ public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map