partitions) {
+ return deleteConsumerGroupOffsets(groupId, partitions, new DeleteConsumerGroupOffsetsOptions());
+ }
+
/**
* Elect the preferred replica as leader for topic partitions.
*
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
index 35a781e1deb5..107eb56d63ae 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
@@ -217,7 +217,7 @@ public static ConfigDef configDef() {
}
public static void main(String[] args) {
- System.out.println(CONFIG.toHtmlTable());
+ System.out.println(CONFIG.toHtml());
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java
index 4fada35c790c..85c0a905240b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/CreateTopicsResult.java
@@ -18,9 +18,11 @@
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.annotation.InterfaceStability;
+import org.apache.kafka.common.errors.ApiException;
import java.util.Collection;
import java.util.Map;
+import java.util.stream.Collectors;
/**
* The result of {@link Admin#createTopics(Collection)}.
@@ -29,9 +31,11 @@
*/
@InterfaceStability.Evolving
public class CreateTopicsResult {
- private final Map> futures;
+ final static int UNKNOWN = -1;
- CreateTopicsResult(Map> futures) {
+ private final Map> futures;
+
+ CreateTopicsResult(Map> futures) {
this.futures = futures;
}
@@ -40,7 +44,8 @@ public class CreateTopicsResult {
* topic creations.
*/
public Map> values() {
- return futures;
+ return futures.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().thenApply(v -> (Void) null)));
}
/**
@@ -49,4 +54,84 @@ public Map> values() {
public KafkaFuture all() {
return KafkaFuture.allOf(futures.values().toArray(new KafkaFuture[0]));
}
+
+ /**
+ * Returns a future that provides topic configs for the topic when the request completes.
+ *
+ * If broker version doesn't support replication factor in the response, throw
+ * {@link org.apache.kafka.common.errors.UnsupportedVersionException}.
+ * If broker returned an error for topic configs, throw appropriate exception. For example,
+ * {@link org.apache.kafka.common.errors.TopicAuthorizationException} is thrown if user does not
+ * have permission to describe topic configs.
+ */
+ public KafkaFuture config(String topic) {
+ return futures.get(topic).thenApply(TopicMetadataAndConfig::config);
+ }
+
+ /**
+ * Returns a future that provides number of partitions in the topic when the request completes.
+ *
+ * If broker version doesn't support replication factor in the response, throw
+ * {@link org.apache.kafka.common.errors.UnsupportedVersionException}.
+ * If broker returned an error for topic configs, throw appropriate exception. For example,
+ * {@link org.apache.kafka.common.errors.TopicAuthorizationException} is thrown if user does not
+ * have permission to describe topic configs.
+ */
+ public KafkaFuture numPartitions(String topic) {
+ return futures.get(topic).thenApply(TopicMetadataAndConfig::numPartitions);
+ }
+
+ /**
+ * Returns a future that provides replication factor for the topic when the request completes.
+ *
+ * If broker version doesn't support replication factor in the response, throw
+ * {@link org.apache.kafka.common.errors.UnsupportedVersionException}.
+ * If broker returned an error for topic configs, throw appropriate exception. For example,
+ * {@link org.apache.kafka.common.errors.TopicAuthorizationException} is thrown if user does not
+ * have permission to describe topic configs.
+ */
+ public KafkaFuture replicationFactor(String topic) {
+ return futures.get(topic).thenApply(TopicMetadataAndConfig::replicationFactor);
+ }
+
+ static class TopicMetadataAndConfig {
+ private final ApiException exception;
+ private final int numPartitions;
+ private final int replicationFactor;
+ private final Config config;
+
+ TopicMetadataAndConfig(int numPartitions, int replicationFactor, Config config) {
+ this.exception = null;
+ this.numPartitions = numPartitions;
+ this.replicationFactor = replicationFactor;
+ this.config = config;
+ }
+
+ TopicMetadataAndConfig(ApiException exception) {
+ this.exception = exception;
+ this.numPartitions = UNKNOWN;
+ this.replicationFactor = UNKNOWN;
+ this.config = null;
+ }
+
+ public int numPartitions() {
+ ensureSuccess();
+ return numPartitions;
+ }
+
+ public int replicationFactor() {
+ ensureSuccess();
+ return replicationFactor;
+ }
+
+ public Config config() {
+ ensureSuccess();
+ return config;
+ }
+
+ private void ensureSuccess() {
+ if (exception != null)
+ throw exception;
+ }
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsOptions.java
new file mode 100644
index 000000000000..63e6b4be84ba
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsOptions.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import java.util.Set;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+/**
+ * Options for the {@link Admin#deleteConsumerGroupOffsets(String, Set)} call.
+ *
+ * The API of this class is evolving, see {@link Admin} for details.
+ */
+@InterfaceStability.Evolving
+public class DeleteConsumerGroupOffsetsOptions extends AbstractOptions {
+
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsResult.java
new file mode 100644
index 000000000000..433f478c3862
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupOffsetsResult.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.admin;
+
+import java.util.Set;
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.KafkaFuture.BaseFunction;
+import org.apache.kafka.common.KafkaFuture.BiConsumer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.annotation.InterfaceStability;
+
+import java.util.Map;
+import org.apache.kafka.common.internals.KafkaFutureImpl;
+import org.apache.kafka.common.protocol.Errors;
+
+/**
+ * The result of the {@link Admin#deleteConsumerGroupOffsets(String, Set)} call.
+ *
+ * The API of this class is evolving, see {@link Admin} for details.
+ */
+@InterfaceStability.Evolving
+public class DeleteConsumerGroupOffsetsResult {
+ private final KafkaFuture> future;
+
+ DeleteConsumerGroupOffsetsResult(KafkaFuture> future) {
+ this.future = future;
+ }
+
+ /**
+ * Return a future which can be used to check the result for a given partition.
+ */
+ public KafkaFuture partitionResult(final TopicPartition partition) {
+ final KafkaFutureImpl result = new KafkaFutureImpl<>();
+
+ this.future.whenComplete(new BiConsumer, Throwable>() {
+ @Override
+ public void accept(final Map topicPartitions, final Throwable throwable) {
+ if (throwable != null) {
+ result.completeExceptionally(throwable);
+ } else if (!topicPartitions.containsKey(partition)) {
+ result.completeExceptionally(new IllegalArgumentException(
+ "Group offset deletion for partition \"" + partition +
+ "\" was not attempted"));
+ } else {
+ final Errors error = topicPartitions.get(partition);
+ if (error == Errors.NONE) {
+ result.complete(null);
+ } else {
+ result.completeExceptionally(error.exception());
+ }
+ }
+
+ }
+ });
+
+ return result;
+ }
+
+ /**
+ * Return a future which succeeds only if all the deletions succeed.
+ */
+ public KafkaFuture all() {
+ return this.future.thenApply(new BaseFunction, Void>() {
+ @Override
+ public Void apply(final Map topicPartitionErrorsMap) {
+ return null;
+ }
+ });
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index c7ea97005a51..a7663da3789d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -25,6 +25,7 @@
import org.apache.kafka.clients.KafkaClient;
import org.apache.kafka.clients.NetworkClient;
import org.apache.kafka.clients.StaleMetadataException;
+import org.apache.kafka.clients.admin.CreateTopicsResult.TopicMetadataAndConfig;
import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResult;
import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults;
import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo;
@@ -68,6 +69,7 @@
import org.apache.kafka.common.message.CreateDelegationTokenResponseData;
import org.apache.kafka.common.message.CreateTopicsRequestData;
import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection;
+import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicConfigs;
import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult;
import org.apache.kafka.common.message.DeleteGroupsRequestData;
import org.apache.kafka.common.message.DeleteTopicsRequestData;
@@ -85,6 +87,10 @@
import org.apache.kafka.common.message.ListGroupsResponseData;
import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData;
import org.apache.kafka.common.message.MetadataRequestData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestPartition;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestTopic;
+import org.apache.kafka.common.message.OffsetDeleteRequestData.OffsetDeleteRequestTopicCollection;
import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
import org.apache.kafka.common.metrics.JmxReporter;
import org.apache.kafka.common.metrics.MetricConfig;
@@ -151,6 +157,9 @@
import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.MetadataResponse;
+import org.apache.kafka.common.requests.OffsetDeleteRequest;
+import org.apache.kafka.common.requests.OffsetDeleteRequest.Builder;
+import org.apache.kafka.common.requests.OffsetDeleteResponse;
import org.apache.kafka.common.requests.OffsetFetchRequest;
import org.apache.kafka.common.requests.OffsetFetchResponse;
import org.apache.kafka.common.requests.RenewDelegationTokenRequest;
@@ -1302,11 +1311,11 @@ int numPendingCalls() {
@Override
public CreateTopicsResult createTopics(final Collection newTopics,
final CreateTopicsOptions options) {
- final Map> topicFutures = new HashMap<>(newTopics.size());
+ final Map> topicFutures = new HashMap<>(newTopics.size());
final CreatableTopicCollection topics = new CreatableTopicCollection();
for (NewTopic newTopic : newTopics) {
if (topicNameIsUnrepresentable(newTopic.name())) {
- KafkaFutureImpl future = new KafkaFutureImpl<>();
+ KafkaFutureImpl future = new KafkaFutureImpl<>();
future.completeExceptionally(new InvalidTopicException("The given topic name '" +
newTopic.name() + "' cannot be represented in a request."));
topicFutures.put(newTopic.name(), future);
@@ -1341,7 +1350,7 @@ public void handleResponse(AbstractResponse abstractResponse) {
}
// Handle server responses for particular topics.
for (CreatableTopicResult result : response.data().topics()) {
- KafkaFutureImpl future = topicFutures.get(result.name());
+ KafkaFutureImpl future = topicFutures.get(result.name());
if (future == null) {
log.warn("Server response mentioned unknown topic {}", result.name());
} else {
@@ -1351,13 +1360,33 @@ public void handleResponse(AbstractResponse abstractResponse) {
if (exception != null) {
future.completeExceptionally(exception);
} else {
- future.complete(null);
+ TopicMetadataAndConfig topicMetadataAndConfig;
+ if (result.topicConfigErrorCode() != Errors.NONE.code()) {
+ topicMetadataAndConfig = new TopicMetadataAndConfig(Errors.forCode(result.topicConfigErrorCode()).exception());
+ } else if (result.numPartitions() == CreateTopicsResult.UNKNOWN) {
+ topicMetadataAndConfig = new TopicMetadataAndConfig(new UnsupportedVersionException(
+ "Topic metadata and configs in CreateTopics response not supported"));
+ } else {
+ List configs = result.configs();
+ Config topicConfig = new Config(configs.stream()
+ .map(config -> new ConfigEntry(config.name(),
+ config.value(),
+ configSource(DescribeConfigsResponse.ConfigSource.forId(config.configSource())),
+ config.isSensitive(),
+ config.readOnly(),
+ Collections.emptyList()))
+ .collect(Collectors.toSet()));
+ topicMetadataAndConfig = new TopicMetadataAndConfig(result.numPartitions(),
+ result.replicationFactor(),
+ topicConfig);
+ }
+ future.complete(topicMetadataAndConfig);
}
}
}
// The server should send back a response for every topic. But do a sanity check anyway.
- for (Map.Entry> entry : topicFutures.entrySet()) {
- KafkaFutureImpl future = entry.getValue();
+ for (Map.Entry> entry : topicFutures.entrySet()) {
+ KafkaFutureImpl future = entry.getValue();
if (!future.isDone()) {
future.completeExceptionally(new ApiException("The server response did not " +
"contain a reference to node " + entry.getKey()));
@@ -3055,6 +3084,92 @@ void handleFailure(Throwable throwable) {
};
}
+ @Override
+ public DeleteConsumerGroupOffsetsResult deleteConsumerGroupOffsets(
+ String groupId,
+ Set partitions,
+ DeleteConsumerGroupOffsetsOptions options) {
+ final KafkaFutureImpl> future = new KafkaFutureImpl<>();
+
+ if (groupIdIsUnrepresentable(groupId)) {
+ future.completeExceptionally(new InvalidGroupIdException("The given group id '" +
+ groupId + "' cannot be represented in a request."));
+ return new DeleteConsumerGroupOffsetsResult(future);
+ }
+
+ final long startFindCoordinatorMs = time.milliseconds();
+ final long deadline = calcDeadlineMs(startFindCoordinatorMs, options.timeoutMs());
+ ConsumerGroupOperationContext, DeleteConsumerGroupOffsetsOptions> context =
+ new ConsumerGroupOperationContext<>(groupId, options, deadline, future);
+
+ Call findCoordinatorCall = getFindCoordinatorCall(context,
+ () -> KafkaAdminClient.this.getDeleteConsumerGroupOffsetsCall(context, partitions));
+ runnable.call(findCoordinatorCall, startFindCoordinatorMs);
+
+ return new DeleteConsumerGroupOffsetsResult(future);
+ }
+
+ private Call getDeleteConsumerGroupOffsetsCall(
+ ConsumerGroupOperationContext, DeleteConsumerGroupOffsetsOptions> context,
+ Set partitions) {
+ return new Call("deleteConsumerGroupOffsets", context.getDeadline(), new ConstantNodeIdProvider(context.getNode().get().id())) {
+
+ @Override
+ AbstractRequest.Builder createRequest(int timeoutMs) {
+ final OffsetDeleteRequestTopicCollection topics = new OffsetDeleteRequestTopicCollection();
+
+ partitions.stream().collect(Collectors.groupingBy(TopicPartition::topic)).forEach((topic, topicPartitions) -> {
+ topics.add(
+ new OffsetDeleteRequestTopic()
+ .setName(topic)
+ .setPartitions(topicPartitions.stream()
+ .map(tp -> new OffsetDeleteRequestPartition().setPartitionIndex(tp.partition()))
+ .collect(Collectors.toList())
+ )
+ );
+ });
+
+ return new OffsetDeleteRequest.Builder(
+ new OffsetDeleteRequestData()
+ .setGroupId(context.groupId)
+ .setTopics(topics)
+ );
+ }
+
+ @Override
+ void handleResponse(AbstractResponse abstractResponse) {
+ final OffsetDeleteResponse response = (OffsetDeleteResponse) abstractResponse;
+
+ // If coordinator changed since we fetched it, retry
+ if (context.hasCoordinatorMoved(response)) {
+ rescheduleTask(context, () -> getDeleteConsumerGroupOffsetsCall(context, partitions));
+ return;
+ }
+
+ // If the error is an error at the group level, the future is failed with it
+ final Errors groupError = Errors.forCode(response.data.errorCode());
+ if (handleGroupRequestError(groupError, context.getFuture()))
+ return;
+
+ final Map partitions = new HashMap<>();
+ response.data.topics().forEach(topic -> {
+ topic.partitions().forEach(partition -> {
+ partitions.put(
+ new TopicPartition(topic.name(), partition.partitionIndex()),
+ Errors.forCode(partition.errorCode()));
+ });
+ });
+
+ context.getFuture().complete(partitions);
+ }
+
+ @Override
+ void handleFailure(Throwable throwable) {
+ context.getFuture().completeExceptionally(throwable);
+ }
+ };
+ }
+
@Override
public Map metrics() {
return Collections.unmodifiableMap(this.metrics.metrics());
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java
index 7fdf4398b9ad..5136d6886dd2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java
@@ -154,13 +154,25 @@ public interface Consumer extends Closeable {
/**
* @see KafkaConsumer#committed(TopicPartition)
*/
+ @Deprecated
OffsetAndMetadata committed(TopicPartition partition);
/**
* @see KafkaConsumer#committed(TopicPartition, Duration)
*/
+ @Deprecated
OffsetAndMetadata committed(TopicPartition partition, final Duration timeout);
+ /**
+ * @see KafkaConsumer#committed(Set)
+ */
+ Map committed(Set partitions);
+
+ /**
+ * @see KafkaConsumer#committed(Set, Duration)
+ */
+ Map committed(Set partitions, final Duration timeout);
+
/**
* @see KafkaConsumer#metrics()
*/
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index a4f3003cc500..e0f7c4d160d5 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -251,9 +251,9 @@ public class ConsumerConfig extends AbstractConfig {
/** isolation.level */
public static final String ISOLATION_LEVEL_CONFIG = "isolation.level";
- public static final String ISOLATION_LEVEL_DOC = "Controls how to read messages written transactionally. If set to read_committed, consumer.poll() will only return" +
+ public static final String ISOLATION_LEVEL_DOC = "Controls how to read messages written transactionally. If set to read_committed, consumer.poll() will only return" +
" transactional messages which have been committed. If set to read_uncommitted' (the default), consumer.poll() will return all messages, even transactional messages" +
- " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.
Messages will always be returned in offset order. Hence, in " +
+ " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.
Messages will always be returned in offset order. Hence, in " +
" read_committed mode, consumer.poll() will only return messages up to the last stable offset (LSO), which is the one less than the offset of the first open transaction." +
" In particular any messages appearing after messages belonging to ongoing transactions will be withheld until the relevant transaction has been completed. As a result, read_committed" +
" consumers will not be able to read up to the high watermark when there are in flight transactions.
Further, when in read_committed the seekToEnd method will" +
@@ -556,7 +556,7 @@ public static ConfigDef configDef() {
}
public static void main(String[] args) {
- System.out.println(CONFIG.toHtmlTable());
+ System.out.println(CONFIG.toHtml());
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index 6046ef954682..ec2ce39a2d9b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -67,7 +67,9 @@
* During a rebalance event, the {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} function will always be triggered exactly once when
* the rebalance completes. That is, even if there is no newly assigned partitions for a consumer member, its {@link #onPartitionsAssigned(Collection) onPartitionsAssigned}
* will still be triggered with an empty collection of partitions. As a result this function can be used also to notify when a rebalance event has happened.
- * On the other hand, {@link #onPartitionsRevoked(Collection)} and {@link #onPartitionsLost(Collection)}
+ * With eager rebalancing, {@link #onPartitionsRevoked(Collection)} will always be called at the start of a rebalance. On the other hand, {@link #onPartitionsLost(Collection)}
+ * will only be called when there were non-empty partitions that were lost.
+ * With cooperative rebalancing, {@link #onPartitionsRevoked(Collection)} and {@link #onPartitionsLost(Collection)}
* will only be triggered when there are non-empty partitions revoked or lost from this consumer member during a rebalance event.
*
* It is possible
@@ -117,16 +119,16 @@
public interface ConsumerRebalanceListener {
/**
- * A callback method the user can implement to provide handling of offset commits to a customized store on the start
- * of a rebalance operation. This method will be called before a rebalance operation starts and after the consumer
- * stops fetching data. It can also be called when consumer is being closed ({@link KafkaConsumer#close(Duration)})
+ * A callback method the user can implement to provide handling of offset commits to a customized store.
+ * This method will be called during a rebalance operation when the consumer has to give up some partitions.
+ * It can also be called when consumer is being closed ({@link KafkaConsumer#close(Duration)})
* or is unsubscribing ({@link KafkaConsumer#unsubscribe()}).
* It is recommended that offsets should be committed in this callback to either Kafka or a
* custom offset store to prevent duplicate data.
*
- * For examples on usage of this API, see Usage Examples section of {@link KafkaConsumer KafkaConsumer}
- *
- * NOTE: This method is only called before rebalances. It is not called prior to {@link KafkaConsumer#close()}.
+ * In eager rebalancing, it will always be called at the start of a rebalance and after the consumer stops fetching data.
+ * In cooperative rebalancing, it will be called at the end of a rebalance on the set of partitions being revoked iff the set is non-empty.
+ * For examples on usage of this API, see Usage Examples section of {@link KafkaConsumer KafkaConsumer}.
*
* It is common for the revocation callback to use the consumer instance in order to commit offsets. It is possible
* for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index be3e176c2eeb..7e023415dc3d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1735,7 +1735,10 @@ public long position(TopicPartition partition, final Duration timeout) {
* @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
* @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
* the timeout specified by {@code default.api.timeout.ms} expires.
+ *
+ * @deprecated since 2.4 Use {@link #committed(Set)} instead
*/
+ @Deprecated
@Override
public OffsetAndMetadata committed(TopicPartition partition) {
return committed(partition, Duration.ofMillis(defaultApiTimeoutMs));
@@ -1745,7 +1748,8 @@ public OffsetAndMetadata committed(TopicPartition partition) {
* Get the last committed offset for the given partition (whether the commit happened by this process or
* another). This offset will be used as the position for the consumer in the event of a failure.
*
- * This call will block to do a remote call to get the latest committed offsets from the server.
+ * This call will block until the position can be determined, an unrecoverable error is
+ * encountered (in which case it is thrown to the caller), or the timeout expires.
*
* @param partition The partition to check
* @param timeout The maximum amount of time to await the current committed offset
@@ -1760,21 +1764,85 @@ public OffsetAndMetadata committed(TopicPartition partition) {
* @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
* @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
* expiration of the timeout
+ *
+ * @deprecated since 2.4 Use {@link #committed(Set, Duration)} instead
*/
+ @Deprecated
@Override
public OffsetAndMetadata committed(TopicPartition partition, final Duration timeout) {
+ return committed(Collections.singleton(partition), timeout).get(partition);
+ }
+
+ /**
+ * Get the last committed offsets for the given partitions (whether the commit happened by this process or
+ * another). The returned offsets will be used as the position for the consumer in the event of a failure.
+ *
+ * Partitions that do not have a committed offset would not be included in the returned map.
+ *
+ * If any of the partitions requested do not exist, an exception would be thrown.
+ *
+ * This call will do a remote call to get the latest committed offsets from the server, and will block until the
+ * committed offsets are gotten successfully, an unrecoverable error is encountered (in which case it is thrown to
+ * the caller), or the timeout specified by {@code default.api.timeout.ms} expires (in which case a
+ * {@link org.apache.kafka.common.errors.TimeoutException} is thrown to the caller).
+ *
+ * @param partitions The partitions to check
+ * @return The latest committed offsets for the given partitions; partitions that do not have any committed offsets
+ * would not be included in the returned result
+ * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this
+ * function is called
+ * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while
+ * this function is called
+ * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
+ * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the
+ * configured groupId. See the exception for more details
+ * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
+ * @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
+ * the timeout specified by {@code default.api.timeout.ms} expires.
+ */
+ @Override
+ public Map committed(final Set partitions) {
+ return committed(partitions, Duration.ofMillis(defaultApiTimeoutMs));
+ }
+
+ /**
+ * Get the last committed offsets for the given partitions (whether the commit happened by this process or
+ * another). The returned offsets will be used as the position for the consumer in the event of a failure.
+ *
+ * Partitions that do not have a committed offset would not be included in the returned map.
+ *
+ * If any of the partitions requested do not exist, an exception would be thrown.
+ *
+ * This call will block to do a remote call to get the latest committed offsets from the server.
+ *
+ * @param partitions The partitions to check
+ * @param timeout The maximum amount of time to await the latest committed offsets
+ * @return The latest committed offsets for the given partitions; partitions that do not have any committed offsets
+ * would not be included in the returned result
+ * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this
+ * function is called
+ * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while
+ * this function is called
+ * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
+ * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the
+ * configured groupId. See the exception for more details
+ * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
+ * @throws org.apache.kafka.common.errors.TimeoutException if the committed offset cannot be found before
+ * expiration of the timeout
+ */
+ @Override
+ public Map committed(final Set partitions, final Duration timeout) {
acquireAndEnsureOpen();
try {
maybeThrowInvalidGroupIdException();
- Map offsets = coordinator.fetchCommittedOffsets(
- Collections.singleton(partition), time.timer(timeout));
+ Map offsets = coordinator.fetchCommittedOffsets(partitions, time.timer(timeout));
if (offsets == null) {
throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the last " +
- "committed offset for partition " + partition + " could be determined. Try tuning default.api.timeout.ms " +
- "larger to relax the threshold.");
+ "committed offset for partitions " + partitions + " could be determined. Try tuning default.api.timeout.ms " +
+ "larger to relax the threshold.");
} else {
offsets.forEach(this::updateLastSeenEpochIfNewer);
- return offsets.get(partition);
+ return offsets;
}
} finally {
release();
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
index 947746f3ce1a..0db883ed1b41 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
@@ -42,6 +42,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
/**
@@ -290,20 +291,33 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata)
subscriptions.seek(partition, offsetAndMetadata.offset());
}
+ @Deprecated
@Override
- public synchronized OffsetAndMetadata committed(TopicPartition partition) {
- ensureNotClosed();
- if (subscriptions.isAssigned(partition)) {
- return committed.get(partition);
- }
- return new OffsetAndMetadata(0);
+ public synchronized OffsetAndMetadata committed(final TopicPartition partition) {
+ return committed(Collections.singleton(partition)).get(partition);
}
+ @Deprecated
@Override
- public OffsetAndMetadata committed(TopicPartition partition, final Duration timeout) {
+ public OffsetAndMetadata committed(final TopicPartition partition, final Duration timeout) {
return committed(partition);
}
+ @Override
+ public synchronized Map committed(final Set partitions) {
+ ensureNotClosed();
+
+ return partitions.stream()
+ .filter(committed::containsKey)
+ .collect(Collectors.toMap(tp -> tp, tp -> subscriptions.isAssigned(tp) ?
+ committed.get(tp) : new OffsetAndMetadata(0)));
+ }
+
+ @Override
+ public synchronized Map committed(final Set partitions, final Duration timeout) {
+ return committed(partitions);
+ }
+
@Override
public synchronized long position(TopicPartition partition) {
ensureNotClosed();
@@ -446,7 +460,6 @@ public synchronized void close() {
@SuppressWarnings("deprecation")
@Override
public synchronized void close(long timeout, TimeUnit unit) {
- ensureNotClosed();
this.closed = true;
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
index 78956abb2182..ea642885c062 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/RangeAssignor.java
@@ -40,6 +40,29 @@
* C0: [t0p0, t0p1, t1p0, t1p1]
* C1: [t0p2, t1p2]
*
+ *
+ * Since the introduction of static membership, we could leverage group.instance.id to make the assignment behavior more sticky.
+ * For the above example, after one rolling bounce, group coordinator will attempt to assign new member.id towards consumers,
+ * for example C0 -> C3 C1 -> C2.
+ *
+ * The assignment could be completely shuffled to:
+ *
+ * C3 (was C0): [t0p2, t1p2] (before was [t0p0, t0p1, t1p0, t1p1])
+ * C2 (was C1): [t0p0, t0p1, t1p0, t1p1] (before was [t0p2, t1p2])
+ *
+ *
+ * The assignment change was caused by the change of member.id relative order, and
+ * can be avoided by setting the group.instance.id.
+ * Consumers will have individual instance ids I1, I2. As long as
+ * 1. Number of members remain the same across generation
+ * 2. Static members' identities persist across generation
+ * 3. Subscription pattern doesn't change for any member
+ *
+ * The assignment will always be:
+ *
+ * I0: [t0p0, t0p1, t1p0, t1p1]
+ * I1: [t0p2, t1p2]
+ *
*/
public class RangeAssignor extends AbstractPartitionAssignor {
@@ -48,27 +71,30 @@ public String name() {
return "range";
}
- private Map> consumersPerTopic(Map consumerMetadata) {
- Map> res = new HashMap<>();
+ private Map> consumersPerTopic(Map consumerMetadata) {
+ Map> topicToConsumers = new HashMap<>();
for (Map.Entry subscriptionEntry : consumerMetadata.entrySet()) {
String consumerId = subscriptionEntry.getKey();
- for (String topic : subscriptionEntry.getValue().topics())
- put(res, topic, consumerId);
+ MemberInfo memberInfo = new MemberInfo(consumerId, subscriptionEntry.getValue().groupInstanceId());
+ for (String topic : subscriptionEntry.getValue().topics()) {
+ put(topicToConsumers, topic, memberInfo);
+ }
}
- return res;
+ return topicToConsumers;
}
@Override
public Map> assign(Map partitionsPerTopic,
Map subscriptions) {
- Map> consumersPerTopic = consumersPerTopic(subscriptions);
+ Map> consumersPerTopic = consumersPerTopic(subscriptions);
+
Map> assignment = new HashMap<>();
for (String memberId : subscriptions.keySet())
assignment.put(memberId, new ArrayList<>());
- for (Map.Entry> topicEntry : consumersPerTopic.entrySet()) {
+ for (Map.Entry> topicEntry : consumersPerTopic.entrySet()) {
String topic = topicEntry.getKey();
- List consumersForTopic = topicEntry.getValue();
+ List consumersForTopic = topicEntry.getValue();
Integer numPartitionsForTopic = partitionsPerTopic.get(topic);
if (numPartitionsForTopic == null)
@@ -83,10 +109,9 @@ public Map> assign(Map partitionsP
for (int i = 0, n = consumersForTopic.size(); i < n; i++) {
int start = numPartitionsPerConsumer * i + Math.min(i, consumersWithExtraPartition);
int length = numPartitionsPerConsumer + (i + 1 > consumersWithExtraPartition ? 0 : 1);
- assignment.get(consumersForTopic.get(i)).addAll(partitions.subList(start, start + length));
+ assignment.get(consumersForTopic.get(i).memberId).addAll(partitions.subList(start, start + length));
}
}
return assignment;
}
-
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
index 0981400a6e5a..0ac90b386834 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/RoundRobinAssignor.java
@@ -77,7 +77,7 @@
* After one rolling bounce, group coordinator will attempt to assign new member.id towards consumers,
* for example C0 -> C5 C1 -> C3, C2 -> C4.
*
- * the assignment could be completely shuffled to:
+ *
The assignment could be completely shuffled to:
*
";
public static final String SASL_ENABLED_MECHANISMS_DOC = "The list of SASL mechanisms enabled in the Kafka server. "
+ "The list may contain any mechanism for which a security provider is available. "
diff --git a/clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java b/clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java
new file mode 100644
index 000000000000..a62fe325d813
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.errors;
+
+public class GroupSubscribedToTopicException extends ApiException {
+ public GroupSubscribedToTopicException(String message) {
+ super(message);
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java b/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java
index 8d7bdbe85a03..3f852cf1a538 100644
--- a/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java
+++ b/clients/src/main/java/org/apache/kafka/common/metrics/JmxReporter.java
@@ -78,6 +78,7 @@ public void init(List metrics) {
public boolean containsMbean(String mbeanName) {
return mbeans.containsKey(mbeanName);
}
+
@Override
public void metricChange(KafkaMetric metric) {
synchronized (LOCK) {
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 2393da202369..ba59940175ea 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -44,6 +44,8 @@
import org.apache.kafka.common.message.InitProducerIdResponseData;
import org.apache.kafka.common.message.JoinGroupRequestData;
import org.apache.kafka.common.message.JoinGroupResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
import org.apache.kafka.common.message.LeaveGroupRequestData;
import org.apache.kafka.common.message.LeaveGroupResponseData;
import org.apache.kafka.common.message.ListGroupsRequestData;
@@ -56,6 +58,8 @@
import org.apache.kafka.common.message.MetadataResponseData;
import org.apache.kafka.common.message.OffsetCommitRequestData;
import org.apache.kafka.common.message.OffsetCommitResponseData;
+import org.apache.kafka.common.message.OffsetDeleteRequestData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
import org.apache.kafka.common.message.OffsetFetchRequestData;
import org.apache.kafka.common.message.OffsetFetchResponseData;
import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
@@ -64,8 +68,12 @@
import org.apache.kafka.common.message.SaslAuthenticateResponseData;
import org.apache.kafka.common.message.SaslHandshakeRequestData;
import org.apache.kafka.common.message.SaslHandshakeResponseData;
+import org.apache.kafka.common.message.StopReplicaRequestData;
+import org.apache.kafka.common.message.StopReplicaResponseData;
import org.apache.kafka.common.message.SyncGroupRequestData;
import org.apache.kafka.common.message.SyncGroupResponseData;
+import org.apache.kafka.common.message.UpdateMetadataRequestData;
+import org.apache.kafka.common.message.UpdateMetadataResponseData;
import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
import org.apache.kafka.common.message.TxnOffsetCommitResponseData;
import org.apache.kafka.common.protocol.types.Schema;
@@ -101,18 +109,12 @@
import org.apache.kafka.common.requests.EndTxnResponse;
import org.apache.kafka.common.requests.FetchRequest;
import org.apache.kafka.common.requests.FetchResponse;
-import org.apache.kafka.common.requests.LeaderAndIsrRequest;
-import org.apache.kafka.common.requests.LeaderAndIsrResponse;
import org.apache.kafka.common.requests.ListOffsetRequest;
import org.apache.kafka.common.requests.ListOffsetResponse;
import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest;
import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse;
import org.apache.kafka.common.requests.ProduceRequest;
import org.apache.kafka.common.requests.ProduceResponse;
-import org.apache.kafka.common.requests.StopReplicaRequest;
-import org.apache.kafka.common.requests.StopReplicaResponse;
-import org.apache.kafka.common.requests.UpdateMetadataRequest;
-import org.apache.kafka.common.requests.UpdateMetadataResponse;
import org.apache.kafka.common.requests.WriteTxnMarkersRequest;
import org.apache.kafka.common.requests.WriteTxnMarkersResponse;
@@ -131,10 +133,9 @@ public enum ApiKeys {
FETCH(1, "Fetch", FetchRequest.schemaVersions(), FetchResponse.schemaVersions()),
LIST_OFFSETS(2, "ListOffsets", ListOffsetRequest.schemaVersions(), ListOffsetResponse.schemaVersions()),
METADATA(3, "Metadata", MetadataRequestData.SCHEMAS, MetadataResponseData.SCHEMAS),
- LEADER_AND_ISR(4, "LeaderAndIsr", true, LeaderAndIsrRequest.schemaVersions(), LeaderAndIsrResponse.schemaVersions()),
- STOP_REPLICA(5, "StopReplica", true, StopReplicaRequest.schemaVersions(), StopReplicaResponse.schemaVersions()),
- UPDATE_METADATA(6, "UpdateMetadata", true, UpdateMetadataRequest.schemaVersions(),
- UpdateMetadataResponse.schemaVersions()),
+ LEADER_AND_ISR(4, "LeaderAndIsr", true, LeaderAndIsrRequestData.SCHEMAS, LeaderAndIsrResponseData.SCHEMAS),
+ STOP_REPLICA(5, "StopReplica", true, StopReplicaRequestData.SCHEMAS, StopReplicaResponseData.SCHEMAS),
+ UPDATE_METADATA(6, "UpdateMetadata", true, UpdateMetadataRequestData.SCHEMAS, UpdateMetadataResponseData.SCHEMAS),
CONTROLLED_SHUTDOWN(7, "ControlledShutdown", true, ControlledShutdownRequestData.SCHEMAS,
ControlledShutdownResponseData.SCHEMAS),
OFFSET_COMMIT(8, "OffsetCommit", OffsetCommitRequestData.SCHEMAS, OffsetCommitResponseData.SCHEMAS),
@@ -201,7 +202,8 @@ public Struct parseResponse(short version, ByteBuffer buffer) {
ALTER_PARTITION_REASSIGNMENTS(45, "AlterPartitionReassignments", AlterPartitionReassignmentsRequestData.SCHEMAS,
AlterPartitionReassignmentsResponseData.SCHEMAS),
LIST_PARTITION_REASSIGNMENTS(46, "ListPartitionReassignments", ListPartitionReassignmentsRequestData.SCHEMAS,
- ListPartitionReassignmentsResponseData.SCHEMAS);
+ ListPartitionReassignmentsResponseData.SCHEMAS),
+ OFFSET_DELETE(47, "OffsetDelete", OffsetDeleteRequestData.SCHEMAS, OffsetDeleteResponseData.SCHEMAS);
private static final ApiKeys[] ID_TO_TYPE;
private static final int MIN_API_KEY = 0;
@@ -332,6 +334,14 @@ public boolean isVersionSupported(short apiVersion) {
return apiVersion >= oldestVersion() && apiVersion <= latestVersion();
}
+ public short headerVersion(short apiVersion) {
+ if ((this == CONTROLLED_SHUTDOWN) && (apiVersion == 0)) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
private static String toHtml() {
final StringBuilder b = new StringBuilder();
b.append("\n");
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
index 9f11fc89adf0..0425ef1a4d31 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java
@@ -20,6 +20,7 @@
import org.apache.kafka.common.errors.BrokerNotAvailableException;
import org.apache.kafka.common.errors.ClusterAuthorizationException;
import org.apache.kafka.common.errors.ConcurrentTransactionsException;
+import org.apache.kafka.common.errors.GroupSubscribedToTopicException;
import org.apache.kafka.common.errors.ControllerMovedException;
import org.apache.kafka.common.errors.CoordinatorLoadInProgressException;
import org.apache.kafka.common.errors.CoordinatorNotAvailableException;
@@ -312,7 +313,9 @@ public enum Errors {
EligibleLeadersNotAvailableException::new),
ELECTION_NOT_NEEDED(84, "Leader election not needed for topic partition", ElectionNotNeededException::new),
NO_REASSIGNMENT_IN_PROGRESS(85, "No partition reassignment is in progress.",
- NoReassignmentInProgressException::new);
+ NoReassignmentInProgressException::new),
+ GROUP_SUBSCRIBED_TO_TOPIC(86, "Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it.",
+ GroupSubscribedToTopicException::new);
private static final Logger log = LoggerFactory.getLogger(Errors.class);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
index e3fdea7feae3..6e31e50c47c2 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java
@@ -19,9 +19,13 @@
import org.apache.kafka.common.utils.Utils;
+import java.nio.ByteBuffer;
import java.util.Iterator;
+import java.util.UUID;
public final class MessageUtil {
+ public static final UUID ZERO_UUID = new UUID(0L, 0L);
+
/**
* Get the length of the UTF8 representation of a string, without allocating
* a byte buffer for the string.
@@ -34,6 +38,21 @@ public static short serializedUtf8Length(CharSequence input) {
return (short) count;
}
+ /**
+ * Copy a byte buffer into an array. This will not affect the buffer's
+ * position or mark.
+ */
+ public static byte[] byteBufferToArray(ByteBuffer buf) {
+ byte[] arr = new byte[buf.remaining()];
+ int prevPosition = buf.position();
+ try {
+ buf.get(arr);
+ } finally {
+ buf.position(prevPosition);
+ }
+ return arr;
+ }
+
public static String deepToString(Iterator iter) {
StringBuilder bld = new StringBuilder("[");
String prefix = "";
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
index b5042c3f92a8..0e0975ccb831 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
@@ -16,12 +16,11 @@
*/
package org.apache.kafka.common.protocol;
-import org.apache.kafka.common.protocol.types.ArrayOf;
+import org.apache.kafka.common.message.RequestHeaderData;
+import org.apache.kafka.common.message.ResponseHeaderData;
import org.apache.kafka.common.protocol.types.BoundField;
import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Type;
-import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.ResponseHeader;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -43,18 +42,19 @@ private static void schemaToBnfHtml(Schema schema, StringBuilder b, int indentSi
// Top level fields
for (BoundField field: schema.fields()) {
- if (field.def.type instanceof ArrayOf) {
+ Type type = field.def.type;
+ if (type.isArray()) {
b.append("[");
b.append(field.def.name);
b.append("] ");
- Type innerType = ((ArrayOf) field.def.type).type();
- if (!subTypes.containsKey(field.def.name))
- subTypes.put(field.def.name, innerType);
+ if (!subTypes.containsKey(field.def.name)) {
+ subTypes.put(field.def.name, type.arrayElementType().get());
+ }
} else {
b.append(field.def.name);
b.append(" ");
if (!subTypes.containsKey(field.def.name))
- subTypes.put(field.def.name, field.def.type);
+ subTypes.put(field.def.name, type);
}
}
b.append("\n");
@@ -81,8 +81,8 @@ private static void schemaToBnfHtml(Schema schema, StringBuilder b, int indentSi
private static void populateSchemaFields(Schema schema, Set fields) {
for (BoundField field: schema.fields()) {
fields.add(field);
- if (field.def.type instanceof ArrayOf) {
- Type innerType = ((ArrayOf) field.def.type).type();
+ if (field.def.type.isArray()) {
+ Type innerType = field.def.type.arrayElementType().get();
if (innerType instanceof Schema)
populateSchemaFields((Schema) innerType, fields);
} else if (field.def.type instanceof Schema)
@@ -116,18 +116,20 @@ public static String toHtml() {
final StringBuilder b = new StringBuilder();
b.append("Headers:
\n");
- b.append("");
- b.append("Request Header => ");
- schemaToBnfHtml(RequestHeader.SCHEMA, b, 2);
- b.append("\n");
- schemaToFieldTableHtml(RequestHeader.SCHEMA, b);
-
- b.append("");
- b.append("Response Header => ");
- schemaToBnfHtml(ResponseHeader.SCHEMA, b, 2);
- b.append("\n");
- schemaToFieldTableHtml(ResponseHeader.SCHEMA, b);
-
+ for (int i = 0; i < RequestHeaderData.SCHEMAS.length; i++) {
+ b.append("");
+ b.append("Request Header v").append(i).append(" => ");
+ schemaToBnfHtml(RequestHeaderData.SCHEMAS[i], b, 2);
+ b.append("\n");
+ schemaToFieldTableHtml(RequestHeaderData.SCHEMAS[i], b);
+ }
+ for (int i = 0; i < ResponseHeaderData.SCHEMAS.length; i++) {
+ b.append("");
+ b.append("Response Header v").append(i).append(" => ");
+ schemaToBnfHtml(ResponseHeaderData.SCHEMAS[i], b, 2);
+ b.append("\n");
+ schemaToFieldTableHtml(ResponseHeaderData.SCHEMAS[i], b);
+ }
for (ApiKeys key : ApiKeys.values()) {
// Key
b.append("");
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
index a527239ca486..e966bde1662a 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java
@@ -18,6 +18,7 @@
package org.apache.kafka.common.protocol;
import java.nio.charset.StandardCharsets;
+import java.util.UUID;
public interface Readable {
byte readByte();
@@ -54,4 +55,11 @@ default byte[] readNullableBytes() {
readArray(arr);
return arr;
}
+
+ /**
+ * Read a UUID with the most significant digits first.
+ */
+ default UUID readUUID() {
+ return new UUID(readLong(), readLong());
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
index 9478ed3ec622..e929e53bc624 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java
@@ -18,6 +18,7 @@
package org.apache.kafka.common.protocol;
import java.nio.charset.StandardCharsets;
+import java.util.UUID;
public interface Writable {
void writeByte(byte val);
@@ -68,4 +69,12 @@ default void writeString(String string) {
writeShort((short) arr.length);
writeArray(arr);
}
+
+ /**
+ * Write a UUID with the most significant digits first.
+ */
+ default void writeUUID(UUID uuid) {
+ writeLong(uuid.getMostSignificantBits());
+ writeLong(uuid.getLeastSignificantBits());
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
index 6609dfd51961..3333084ef663 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
@@ -19,6 +19,7 @@
import org.apache.kafka.common.protocol.types.Type.DocumentedType;
import java.nio.ByteBuffer;
+import java.util.Optional;
/**
* Represents a type for an array of a particular type
@@ -91,8 +92,9 @@ public int sizeOf(Object o) {
return size;
}
- public Type type() {
- return type;
+ @Override
+ public Optional arrayElementType() {
+ return Optional.of(type);
}
@Override
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
index 72e051c179e7..b79bc932b420 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Field.java
@@ -75,6 +75,16 @@ public Int64(String name, String docString, long defaultValue) {
}
}
+ public static class UUID extends Field {
+ public UUID(String name, String docString) {
+ super(name, Type.UUID, docString, false, null);
+ }
+
+ public UUID(String name, String docString, UUID defaultValue) {
+ super(name, Type.UUID, docString, true, defaultValue);
+ }
+ }
+
public static class Int16 extends Field {
public Int16(String name, String docString) {
super(name, Type.INT16, docString, false, null);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
index 5b12eee6e7a8..721b8c63bd76 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
@@ -209,10 +209,9 @@ private static void handleNode(Type node, Visitor visitor) {
visitor.visit(schema);
for (BoundField f : schema.fields())
handleNode(f.def.type, visitor);
- } else if (node instanceof ArrayOf) {
- ArrayOf array = (ArrayOf) node;
- visitor.visit(array);
- handleNode(array.type(), visitor);
+ } else if (node.isArray()) {
+ visitor.visit(node);
+ handleNode(node.arrayElementType().get(), visitor);
} else {
visitor.visit(node);
}
@@ -223,7 +222,6 @@ private static void handleNode(Type node, Visitor visitor) {
*/
public static abstract class Visitor {
public void visit(Schema schema) {}
- public void visit(ArrayOf array) {}
public void visit(Type field) {}
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
index e47a2cdce2a6..0b69851f1644 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
@@ -21,6 +21,7 @@
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Objects;
+import java.util.UUID;
/**
* A record that can be serialized and deserialized according to a pre-defined schema
@@ -88,6 +89,10 @@ public Long get(Field.Int64 field) {
return getLong(field.name);
}
+ public UUID get(Field.UUID field) {
+ return getUUID(field.name);
+ }
+
public Short get(Field.Int16 field) {
return getShort(field.name);
}
@@ -118,6 +123,12 @@ public Long getOrElse(Field.Int64 field, long alternative) {
return alternative;
}
+ public UUID getOrElse(Field.UUID field, UUID alternative) {
+ if (hasField(field.name))
+ return getUUID(field.name);
+ return alternative;
+ }
+
public Short getOrElse(Field.Int16 field, short alternative) {
if (hasField(field.name))
return getShort(field.name);
@@ -245,6 +256,14 @@ public Long getLong(String name) {
return (Long) get(name);
}
+ public UUID getUUID(BoundField field) {
+ return (UUID) get(field);
+ }
+
+ public UUID getUUID(String name) {
+ return (UUID) get(name);
+ }
+
public Object[] getArray(BoundField field) {
return (Object[]) get(field);
}
@@ -342,6 +361,10 @@ public Struct set(Field.Int64 def, long value) {
return set(def.name, value);
}
+ public Struct set(Field.UUID def, UUID value) {
+ return set(def.name, value);
+ }
+
public Struct set(Field.Int16 def, short value) {
return set(def.name, value);
}
@@ -395,9 +418,8 @@ public Struct instance(BoundField field) {
validateField(field);
if (field.def.type instanceof Schema) {
return new Struct((Schema) field.def.type);
- } else if (field.def.type instanceof ArrayOf) {
- ArrayOf array = (ArrayOf) field.def.type;
- return new Struct((Schema) array.type());
+ } else if (field.def.type.isArray()) {
+ return new Struct((Schema) field.def.type.arrayElementType().get());
} else {
throw new SchemaException("Field '" + field.def.name + "' is not a container type, it is of type " + field.def.type);
}
@@ -449,6 +471,7 @@ public void writeTo(ByteBuffer buffer) {
* @throws SchemaException If validation fails
*/
private void validateField(BoundField field) {
+ Objects.requireNonNull(field, "`field` must be non-null");
if (this.schema != field.schema)
throw new SchemaException("Attempt to access field '" + field.def.name + "' from a different schema instance.");
if (field.index > values.length)
@@ -472,7 +495,7 @@ public String toString() {
BoundField f = this.schema.get(i);
b.append(f.def.name);
b.append('=');
- if (f.def.type instanceof ArrayOf && this.values[i] != null) {
+ if (f.def.type.isArray() && this.values[i] != null) {
Object[] arrayValue = (Object[]) this.values[i];
b.append('[');
for (int j = 0; j < arrayValue.length; j++) {
@@ -496,7 +519,7 @@ public int hashCode() {
int result = 1;
for (int i = 0; i < this.values.length; i++) {
BoundField f = this.schema.get(i);
- if (f.def.type instanceof ArrayOf) {
+ if (f.def.type.isArray()) {
if (this.get(f) != null) {
Object[] arrayObject = (Object[]) this.get(f);
for (Object arrayItem: arrayObject)
@@ -526,7 +549,7 @@ public boolean equals(Object obj) {
for (int i = 0; i < this.values.length; i++) {
BoundField f = this.schema.get(i);
boolean result;
- if (f.def.type instanceof ArrayOf) {
+ if (f.def.type.isArray()) {
result = Arrays.equals((Object[]) this.get(f), (Object[]) other.get(f));
} else {
Object thisField = this.get(f);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
index 4bd508b77989..aa0ee40e6809 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
@@ -23,6 +23,8 @@
import org.apache.kafka.common.utils.Utils;
import java.nio.ByteBuffer;
+import java.util.Optional;
+import java.util.UUID;
/**
* A serializable type
@@ -63,6 +65,20 @@ public boolean isNullable() {
return false;
}
+ /**
+ * If the type is an array, return the type of the array elements. Otherwise, return empty.
+ */
+ public Optional arrayElementType() {
+ return Optional.empty();
+ }
+
+ /**
+ * Returns true if the type is an array.
+ */
+ public final boolean isArray() {
+ return arrayElementType().isPresent();
+ }
+
/**
* A Type that can return its description for documentation purposes.
*/
@@ -313,6 +329,44 @@ public String documentation() {
}
};
+ public static final DocumentedType UUID = new DocumentedType() {
+ @Override
+ public void write(ByteBuffer buffer, Object o) {
+ final java.util.UUID uuid = (java.util.UUID) o;
+ buffer.putLong(uuid.getMostSignificantBits());
+ buffer.putLong(uuid.getLeastSignificantBits());
+ }
+
+ @Override
+ public Object read(ByteBuffer buffer) {
+ return new java.util.UUID(buffer.getLong(), buffer.getLong());
+ }
+
+ @Override
+ public int sizeOf(Object o) {
+ return 16;
+ }
+
+ @Override
+ public String typeName() {
+ return "UUID";
+ }
+
+ @Override
+ public UUID validate(Object item) {
+ if (item instanceof UUID)
+ return (UUID) item;
+ else
+ throw new SchemaException(item + " is not a UUID.");
+ }
+
+ @Override
+ public String documentation() {
+ return "Represents a java.util.UUID. " +
+ "The values are encoded using sixteen bytes in network byte order (big-endian).";
+ }
+ };
+
public static final DocumentedType STRING = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
@@ -665,10 +719,9 @@ public String documentation() {
};
private static String toHtml() {
-
DocumentedType[] types = {
BOOLEAN, INT8, INT16, INT32, INT64,
- UNSIGNED_INT32, VARINT, VARLONG,
+ UNSIGNED_INT32, VARINT, VARLONG, UUID,
STRING, NULLABLE_STRING, BYTES, NULLABLE_BYTES,
RECORDS, new ArrayOf(STRING)};
final StringBuilder b = new StringBuilder();
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
index d1fae0502c43..dc4a1e21e8dd 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
@@ -17,20 +17,11 @@
package org.apache.kafka.common.requests;
import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Struct;
// Abstract class for all control requests including UpdateMetadataRequest, LeaderAndIsrRequest and StopReplicaRequest
public abstract class AbstractControlRequest extends AbstractRequest {
- public static final long UNKNOWN_BROKER_EPOCH = -1L;
-
- protected static final Field.Int32 CONTROLLER_ID = new Field.Int32("controller_id", "The controller id");
- protected static final Field.Int32 CONTROLLER_EPOCH = new Field.Int32("controller_epoch", "The controller epoch");
- protected static final Field.Int64 BROKER_EPOCH = new Field.Int64("broker_epoch", "The broker epoch");
- protected final int controllerId;
- protected final int controllerEpoch;
- protected final long brokerEpoch;
+ public static final long UNKNOWN_BROKER_EPOCH = -1L;
public static abstract class Builder extends AbstractRequest.Builder {
protected final int controllerId;
@@ -46,35 +37,14 @@ protected Builder(ApiKeys api, short version, int controllerId, int controllerEp
}
- public int controllerId() {
- return controllerId;
- }
-
- public int controllerEpoch() {
- return controllerEpoch;
- }
-
- public long brokerEpoch() {
- return brokerEpoch;
- }
-
- protected AbstractControlRequest(ApiKeys api, short version, int controllerId, int controllerEpoch, long brokerEpoch) {
+ protected AbstractControlRequest(ApiKeys api, short version) {
super(api, version);
- this.controllerId = controllerId;
- this.controllerEpoch = controllerEpoch;
- this.brokerEpoch = brokerEpoch;
}
- protected AbstractControlRequest(ApiKeys api, Struct struct, short version) {
- super(api, version);
- this.controllerId = struct.get(CONTROLLER_ID);
- this.controllerEpoch = struct.get(CONTROLLER_EPOCH);
- this.brokerEpoch = struct.getOrElse(BROKER_EPOCH, UNKNOWN_BROKER_EPOCH);
- }
+ public abstract int controllerId();
- // Used for test
- long size() {
- return toStruct().sizeOf();
- }
+ public abstract int controllerEpoch();
+
+ public abstract long brokerEpoch();
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
index 58bf1283a77a..b737b49f4a5c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
@@ -237,6 +237,8 @@ public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Str
return new AlterPartitionReassignmentsRequest(struct, apiVersion);
case LIST_PARTITION_REASSIGNMENTS:
return new ListPartitionReassignmentsRequest(struct, apiVersion);
+ case OFFSET_DELETE:
+ return new OffsetDeleteRequest(struct, apiVersion);
default:
throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " +
"code should be updated to do so.", apiKey));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 11466d724986..eb11a4244215 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -23,6 +23,7 @@
import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -31,14 +32,22 @@ public abstract class AbstractResponse extends AbstractRequestResponse {
public static final int DEFAULT_THROTTLE_TIME = 0;
protected Send toSend(String destination, ResponseHeader header, short apiVersion) {
- return new NetworkSend(destination, serialize(apiVersion, header));
+ return new NetworkSend(destination, serialize(header.toStruct(), toStruct(apiVersion)));
}
/**
* Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead.
*/
- public ByteBuffer serialize(short version, ResponseHeader responseHeader) {
- return serialize(responseHeader.toStruct(), toStruct(version));
+ public ByteBuffer serialize(ApiKeys apiKey, int correlationId) {
+ return serialize(apiKey, apiKey.latestVersion(), correlationId);
+ }
+
+ /**
+ * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead.
+ */
+ public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) {
+ ResponseHeader header = new ResponseHeader(correlationId, apiKey.headerVersion(version));
+ return serialize(header.toStruct(), toStruct(version));
}
public abstract Map errorCounts();
@@ -47,9 +56,9 @@ protected Map errorCounts(Errors error) {
return Collections.singletonMap(error, 1);
}
- protected Map errorCounts(Map errors) {
+ protected Map errorCounts(Collection errors) {
Map errorCounts = new HashMap<>();
- for (Errors error : errors.values())
+ for (Errors error : errors)
updateErrorCounts(errorCounts, error);
return errorCounts;
}
@@ -93,13 +102,13 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
case SYNC_GROUP:
return new SyncGroupResponse(struct, version);
case STOP_REPLICA:
- return new StopReplicaResponse(struct);
+ return new StopReplicaResponse(struct, version);
case CONTROLLED_SHUTDOWN:
return new ControlledShutdownResponse(struct, version);
case UPDATE_METADATA:
- return new UpdateMetadataResponse(struct);
+ return new UpdateMetadataResponse(struct, version);
case LEADER_AND_ISR:
- return new LeaderAndIsrResponse(struct);
+ return new LeaderAndIsrResponse(struct, version);
case DESCRIBE_GROUPS:
return new DescribeGroupsResponse(struct, version);
case LIST_GROUPS:
@@ -164,6 +173,8 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor
return new AlterPartitionReassignmentsResponse(struct, version);
case LIST_PARTITION_REASSIGNMENTS:
return new ListPartitionReassignmentsResponse(struct, version);
+ case OFFSET_DELETE:
+ return new OffsetDeleteResponse(struct, version);
default:
throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " +
"code should be updated to do so.", apiKey));
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
index ea8a073e0175..7f20f07b6a11 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
@@ -102,7 +102,7 @@ public Map errors() {
@Override
public Map errorCounts() {
- return errorCounts(errors);
+ return errorCounts(errors.values());
}
@Override
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
index 7a73275d7457..2bbdce2122bc 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java
@@ -132,7 +132,7 @@ public Map responses() {
@Override
public Map errorCounts() {
- return errorCounts(responses);
+ return errorCounts(responses.values());
}
public static AlterReplicaLogDirsResponse parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
index f89316aa3093..83e9444da0ad 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
@@ -64,18 +64,8 @@ public ControlledShutdownRequest(Struct struct, short version) {
@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- ControlledShutdownResponseData response = new ControlledShutdownResponseData();
- response.setErrorCode(Errors.forException(e).code());
- short versionId = version();
- switch (versionId) {
- case 0:
- case 1:
- case 2:
- return new ControlledShutdownResponse(response);
- default:
- throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
- versionId, this.getClass().getSimpleName(), ApiKeys.CONTROLLED_SHUTDOWN.latestVersion()));
- }
+ return new ControlledShutdownResponse(new ControlledShutdownResponseData().
+ setErrorCode(Errors.forException(e).code()));
}
public static ControlledShutdownRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
index d3061c484489..2ad947c5904b 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java
@@ -61,26 +61,18 @@ public DeleteGroupsRequest(Struct struct, short version) {
@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
Errors error = Errors.forException(e);
-
- switch (version()) {
- case 0:
- case 1:
- DeletableGroupResultCollection groupResults = new DeletableGroupResultCollection();
- for (String groupId : data.groupsNames()) {
- groupResults.add(new DeletableGroupResult()
- .setGroupId(groupId)
- .setErrorCode(error.code()));
- }
-
- return new DeleteGroupsResponse(
- new DeleteGroupsResponseData()
- .setResults(groupResults)
- .setThrottleTimeMs(throttleTimeMs)
- );
- default:
- throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
- version(), ApiKeys.DELETE_GROUPS.name, ApiKeys.DELETE_GROUPS.latestVersion()));
+ DeletableGroupResultCollection groupResults = new DeletableGroupResultCollection();
+ for (String groupId : data.groupsNames()) {
+ groupResults.add(new DeletableGroupResult()
+ .setGroupId(groupId)
+ .setErrorCode(error.code()));
}
+
+ return new DeleteGroupsResponse(
+ new DeleteGroupsResponseData()
+ .setResults(groupResults)
+ .setThrottleTimeMs(throttleTimeMs)
+ );
}
public static DeleteGroupsRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java
index 5fe86eb8f164..a98a7c881d96 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java
@@ -86,7 +86,7 @@ public Map errorCounts() {
}
public static DeleteGroupsResponse parse(ByteBuffer buffer, short version) {
- return new DeleteGroupsResponse(ApiKeys.DELETE_GROUPS.parseResponse(version, buffer));
+ return new DeleteGroupsResponse(ApiKeys.DELETE_GROUPS.parseResponse(version, buffer), version);
}
@Override
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
index ff034df34f13..e5cc6c8860a5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
@@ -65,21 +65,12 @@ public HeartbeatRequest(Struct struct, short version) {
@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- HeartbeatResponseData response = new HeartbeatResponseData();
- response.setErrorCode(Errors.forException(e).code());
- short versionId = version();
- switch (versionId) {
- case 0:
- return new HeartbeatResponse(response);
- case 1:
- case 2:
- case 3:
- response.setThrottleTimeMs(throttleTimeMs);
- return new HeartbeatResponse(response);
- default:
- throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
- versionId, this.getClass().getSimpleName(), ApiKeys.HEARTBEAT.latestVersion()));
+ HeartbeatResponseData responseData = new HeartbeatResponseData().
+ setErrorCode(Errors.forException(e).code());
+ if (version() >= 1) {
+ responseData.setThrottleTimeMs(throttleTimeMs);
}
+ return new HeartbeatResponse(responseData);
}
public static HeartbeatRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index 02c14e3274a9..b799e360bd4f 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -17,152 +17,37 @@
package org.apache.kafka.common.requests;
import org.apache.kafka.common.Node;
-import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrTopicState;
+import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError;
import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.utils.CollectionUtils;
+import org.apache.kafka.common.utils.FlattenedIterator;
import org.apache.kafka.common.utils.Utils;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.Collections;
-
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
+import java.util.stream.Collectors;
public class LeaderAndIsrRequest extends AbstractControlRequest {
- private static final Field.ComplexArray TOPIC_STATES = new Field.ComplexArray("topic_states", "Topic states");
- private static final Field.ComplexArray PARTITION_STATES = new Field.ComplexArray("partition_states", "Partition states");
- private static final Field.ComplexArray LIVE_LEADERS = new Field.ComplexArray("live_leaders", "Live leaders");
-
- // PartitionState fields
- private static final Field.Int32 LEADER = new Field.Int32("leader", "The broker id for the leader.");
- private static final Field.Int32 LEADER_EPOCH = new Field.Int32("leader_epoch", "The leader epoch.");
- private static final Field.Array ISR = new Field.Array("isr", INT32, "The in sync replica ids.");
- private static final Field.Int32 ZK_VERSION = new Field.Int32("zk_version", "The ZK version.");
- private static final Field.Array REPLICAS = new Field.Array("replicas", INT32, "The replica ids.");
- private static final Field.Array ADDING_REPLICAS = new Field.Array("adding_replicas", INT32,
- "The replica ids we are in the process of adding to the replica set during a reassignment.");
- private static final Field.Array REMOVING_REPLICAS = new Field.Array("removing_replicas", INT32,
- "The replica ids we are in the process of removing from the replica set during a reassignment.");
- private static final Field.Bool IS_NEW = new Field.Bool("is_new", "Whether the replica should have existed on the broker or not");
-
- // live_leaders fields
- private static final Field.Int32 END_POINT_ID = new Field.Int32("id", "The broker id");
- private static final Field.Str HOST = new Field.Str("host", "The hostname of the broker.");
- private static final Field.Int32 PORT = new Field.Int32("port", "The port on which the broker accepts requests.");
-
- private static final Field PARTITION_STATES_V0 = PARTITION_STATES.withFields(
- TOPIC_NAME,
- PARTITION_ID,
- CONTROLLER_EPOCH,
- LEADER,
- LEADER_EPOCH,
- ISR,
- ZK_VERSION,
- REPLICAS);
-
- // PARTITION_STATES_V1 added a per-partition is_new Field.
- // This field specifies whether the replica should have existed on the broker or not.
- private static final Field PARTITION_STATES_V1 = PARTITION_STATES.withFields(
- TOPIC_NAME,
- PARTITION_ID,
- CONTROLLER_EPOCH,
- LEADER,
- LEADER_EPOCH,
- ISR,
- ZK_VERSION,
- REPLICAS,
- IS_NEW);
-
- private static final Field PARTITION_STATES_V2 = PARTITION_STATES.withFields(
- PARTITION_ID,
- CONTROLLER_EPOCH,
- LEADER,
- LEADER_EPOCH,
- ISR,
- ZK_VERSION,
- REPLICAS,
- IS_NEW);
-
- private static final Field PARTITION_STATES_V3 = PARTITION_STATES.withFields(
- PARTITION_ID,
- CONTROLLER_EPOCH,
- LEADER,
- LEADER_EPOCH,
- ISR,
- ZK_VERSION,
- REPLICAS,
- ADDING_REPLICAS,
- REMOVING_REPLICAS,
- IS_NEW);
-
- // TOPIC_STATES_V2 normalizes TOPIC_STATES_V1 to make it more memory efficient
- private static final Field TOPIC_STATES_V2 = TOPIC_STATES.withFields(
- TOPIC_NAME,
- PARTITION_STATES_V2);
-
- // TOPIC_STATES_V3 adds two new fields - adding_replicas and removing_replicas
- private static final Field TOPIC_STATES_V3 = TOPIC_STATES.withFields(
- TOPIC_NAME,
- PARTITION_STATES_V3);
-
- private static final Field LIVE_LEADERS_V0 = LIVE_LEADERS.withFields(
- END_POINT_ID,
- HOST,
- PORT);
-
- private static final Schema LEADER_AND_ISR_REQUEST_V0 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- PARTITION_STATES_V0,
- LIVE_LEADERS_V0);
-
- // LEADER_AND_ISR_REQUEST_V1 added a per-partition is_new Field. This field specifies whether the replica should
- // have existed on the broker or not.
- private static final Schema LEADER_AND_ISR_REQUEST_V1 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- PARTITION_STATES_V1,
- LIVE_LEADERS_V0);
-
- // LEADER_AND_ISR_REQUEST_V2 added a broker_epoch Field. This field specifies the generation of the broker across
- // bounces. It also normalizes partitions under each topic.
- private static final Schema LEADER_AND_ISR_REQUEST_V2 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- BROKER_EPOCH,
- TOPIC_STATES_V2,
- LIVE_LEADERS_V0);
-
- // LEADER_AND_ISR_REQUEST_V3 added two new fields - adding_replicas and removing_replicas.
- // These fields respectively specify the replica IDs we want to add or remove as part of a reassignment
- private static final Schema LEADER_AND_ISR_REQUEST_V3 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- BROKER_EPOCH,
- TOPIC_STATES_V3,
- LIVE_LEADERS_V0);
-
- public static Schema[] schemaVersions() {
- return new Schema[]{LEADER_AND_ISR_REQUEST_V0, LEADER_AND_ISR_REQUEST_V1, LEADER_AND_ISR_REQUEST_V2, LEADER_AND_ISR_REQUEST_V3};
- }
public static class Builder extends AbstractControlRequest.Builder {
- private final Map partitionStates;
- private final Set liveLeaders;
+
+ private final List partitionStates;
+ private final Collection liveLeaders;
public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- Map partitionStates, Set liveLeaders) {
+ List partitionStates, Collection liveLeaders) {
super(ApiKeys.LEADER_AND_ISR, version, controllerId, controllerEpoch, brokerEpoch);
this.partitionStates = partitionStates;
this.liveLeaders = liveLeaders;
@@ -170,7 +55,38 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
@Override
public LeaderAndIsrRequest build(short version) {
- return new LeaderAndIsrRequest(controllerId, controllerEpoch, brokerEpoch, partitionStates, liveLeaders, version);
+ List leaders = liveLeaders.stream().map(n -> new LeaderAndIsrLiveLeader()
+ .setBrokerId(n.id())
+ .setHostName(n.host())
+ .setPort(n.port())
+ ).collect(Collectors.toList());
+
+ LeaderAndIsrRequestData data = new LeaderAndIsrRequestData()
+ .setControllerId(controllerId)
+ .setControllerEpoch(controllerEpoch)
+ .setBrokerEpoch(brokerEpoch)
+ .setLiveLeaders(leaders);
+
+ if (version >= 2) {
+ Map topicStatesMap = groupByTopic(partitionStates);
+ data.setTopicStates(new ArrayList<>(topicStatesMap.values()));
+ } else {
+ data.setUngroupedPartitionStates(partitionStates);
+ }
+
+ return new LeaderAndIsrRequest(data, version);
+ }
+
+ private static Map groupByTopic(List partitionStates) {
+ Map topicStates = new HashMap<>();
+ // We don't null out the topic name in LeaderAndIsrRequestPartition since it's ignored by
+ // the generated code if version >= 2
+ for (LeaderAndIsrPartitionState partition : partitionStates) {
+ LeaderAndIsrTopicState topicState = topicStates.computeIfAbsent(partition.topicName(),
+ t -> new LeaderAndIsrTopicState().setTopicName(partition.topicName()));
+ topicState.partitionStates().add(partition);
+ }
+ return topicStates;
}
@Override
@@ -184,116 +100,60 @@ public String toString() {
.append(", liveLeaders=(").append(Utils.join(liveLeaders, ", ")).append(")")
.append(")");
return bld.toString();
+
}
}
- private final Map partitionStates;
- private final Set liveLeaders;
+ private final LeaderAndIsrRequestData data;
- private LeaderAndIsrRequest(int controllerId, int controllerEpoch, long brokerEpoch, Map partitionStates,
- Set liveLeaders, short version) {
- super(ApiKeys.LEADER_AND_ISR, version, controllerId, controllerEpoch, brokerEpoch);
- this.partitionStates = partitionStates;
- this.liveLeaders = liveLeaders;
+ LeaderAndIsrRequest(LeaderAndIsrRequestData data, short version) {
+ super(ApiKeys.LEADER_AND_ISR, version);
+ this.data = data;
+ // Do this from the constructor to make it thread-safe (even though it's only needed when some methods are called)
+ normalize();
}
- public LeaderAndIsrRequest(Struct struct, short version) {
- super(ApiKeys.LEADER_AND_ISR, struct, version);
-
- Map partitionStates = new HashMap<>();
- if (struct.hasField(TOPIC_STATES)) {
- for (Object topicStatesDataObj : struct.get(TOPIC_STATES)) {
- Struct topicStatesData = (Struct) topicStatesDataObj;
- String topic = topicStatesData.get(TOPIC_NAME);
- for (Object partitionStateDataObj : topicStatesData.get(PARTITION_STATES)) {
- Struct partitionStateData = (Struct) partitionStateDataObj;
- int partition = partitionStateData.get(PARTITION_ID);
- PartitionState partitionState = new PartitionState(partitionStateData);
- partitionStates.put(new TopicPartition(topic, partition), partitionState);
+ private void normalize() {
+ if (version() >= 2) {
+ for (LeaderAndIsrTopicState topicState : data.topicStates()) {
+ for (LeaderAndIsrPartitionState partitionState : topicState.partitionStates()) {
+ // Set the topic name so that we can always present the ungrouped view to callers
+ partitionState.setTopicName(topicState.topicName());
}
}
- } else {
- for (Object partitionStateDataObj : struct.get(PARTITION_STATES)) {
- Struct partitionStateData = (Struct) partitionStateDataObj;
- String topic = partitionStateData.get(TOPIC_NAME);
- int partition = partitionStateData.get(PARTITION_ID);
- PartitionState partitionState = new PartitionState(partitionStateData);
- partitionStates.put(new TopicPartition(topic, partition), partitionState);
- }
- }
-
- Set leaders = new HashSet<>();
- for (Object leadersDataObj : struct.get(LIVE_LEADERS)) {
- Struct leadersData = (Struct) leadersDataObj;
- int id = leadersData.get(END_POINT_ID);
- String host = leadersData.get(HOST);
- int port = leadersData.get(PORT);
- leaders.add(new Node(id, host, port));
}
+ }
- this.partitionStates = partitionStates;
- this.liveLeaders = leaders;
+ public LeaderAndIsrRequest(Struct struct, short version) {
+ this(new LeaderAndIsrRequestData(struct, version), version);
}
@Override
protected Struct toStruct() {
- short version = version();
- Struct struct = new Struct(ApiKeys.LEADER_AND_ISR.requestSchema(version));
- struct.set(CONTROLLER_ID, controllerId);
- struct.set(CONTROLLER_EPOCH, controllerEpoch);
- struct.setIfExists(BROKER_EPOCH, brokerEpoch);
-
- if (struct.hasField(TOPIC_STATES)) {
- Map> topicStates = CollectionUtils.groupPartitionDataByTopic(partitionStates);
- List topicStatesData = new ArrayList<>(topicStates.size());
- for (Map.Entry> entry : topicStates.entrySet()) {
- Struct topicStateData = struct.instance(TOPIC_STATES);
- topicStateData.set(TOPIC_NAME, entry.getKey());
- Map partitionMap = entry.getValue();
- List partitionStatesData = new ArrayList<>(partitionMap.size());
- for (Map.Entry partitionEntry : partitionMap.entrySet()) {
- Struct partitionStateData = topicStateData.instance(PARTITION_STATES);
- partitionStateData.set(PARTITION_ID, partitionEntry.getKey());
- partitionEntry.getValue().setStruct(partitionStateData, version);
- partitionStatesData.add(partitionStateData);
- }
- topicStateData.set(PARTITION_STATES, partitionStatesData.toArray());
- topicStatesData.add(topicStateData);
- }
- struct.set(TOPIC_STATES, topicStatesData.toArray());
- } else {
- List partitionStatesData = new ArrayList<>(partitionStates.size());
- for (Map.Entry entry : partitionStates.entrySet()) {
- Struct partitionStateData = struct.instance(PARTITION_STATES);
- TopicPartition topicPartition = entry.getKey();
- partitionStateData.set(TOPIC_NAME, topicPartition.topic());
- partitionStateData.set(PARTITION_ID, topicPartition.partition());
- entry.getValue().setStruct(partitionStateData, version);
- partitionStatesData.add(partitionStateData);
- }
- struct.set(PARTITION_STATES, partitionStatesData.toArray());
- }
+ return data.toStruct(version());
+ }
- List leadersData = new ArrayList<>(liveLeaders.size());
- for (Node leader : liveLeaders) {
- Struct leaderData = struct.instance(LIVE_LEADERS);
- leaderData.set(END_POINT_ID, leader.id());
- leaderData.set(HOST, leader.host());
- leaderData.set(PORT, leader.port());
- leadersData.add(leaderData);
- }
- struct.set(LIVE_LEADERS, leadersData.toArray());
- return struct;
+ protected ByteBuffer toBytes() {
+ ByteBuffer bytes = ByteBuffer.allocate(size());
+ data.write(new ByteBufferAccessor(bytes), version());
+ bytes.flip();
+ return bytes;
}
@Override
public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+ LeaderAndIsrResponseData responseData = new LeaderAndIsrResponseData();
Errors error = Errors.forException(e);
-
- Map responses = new HashMap<>(partitionStates.size());
- for (TopicPartition partition : partitionStates.keySet()) {
- responses.put(partition, error);
+ responseData.setErrorCode(error.code());
+
+ List partitions = new ArrayList<>();
+ for (LeaderAndIsrPartitionState partition : partitionStates()) {
+ partitions.add(new LeaderAndIsrPartitionError()
+ .setTopicName(partition.topicName())
+ .setPartitionIndex(partition.partitionIndex())
+ .setErrorCode(error.code()));
}
+ responseData.setPartitionErrors(partitions);
short versionId = version();
switch (versionId) {
@@ -301,133 +161,45 @@ public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
case 1:
case 2:
case 3:
- return new LeaderAndIsrResponse(error, responses);
+ return new LeaderAndIsrResponse(responseData);
default:
throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
versionId, this.getClass().getSimpleName(), ApiKeys.LEADER_AND_ISR.latestVersion()));
}
}
+ @Override
public int controllerId() {
- return controllerId;
+ return data.controllerId();
}
+ @Override
public int controllerEpoch() {
- return controllerEpoch;
+ return data.controllerEpoch();
}
- public Map partitionStates() {
- return partitionStates;
+ @Override
+ public long brokerEpoch() {
+ return data.brokerEpoch();
}
- public Set liveLeaders() {
- return liveLeaders;
+ public Iterable partitionStates() {
+ if (version() >= 2)
+ return () -> new FlattenedIterator<>(data.topicStates().iterator(),
+ topicState -> topicState.partitionStates().iterator());
+ return data.ungroupedPartitionStates();
}
- public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) {
- return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version);
+ public List liveLeaders() {
+ return Collections.unmodifiableList(data.liveLeaders());
}
- public static final class PartitionState {
- public final BasePartitionState basePartitionState;
- public final List addingReplicas;
- public final List removingReplicas;
- public final boolean isNew;
-
- public PartitionState(int controllerEpoch,
- int leader,
- int leaderEpoch,
- List isr,
- int zkVersion,
- List replicas,
- boolean isNew) {
- this(controllerEpoch,
- leader,
- leaderEpoch,
- isr,
- zkVersion,
- replicas,
- Collections.emptyList(),
- Collections.emptyList(),
- isNew);
- }
-
- public PartitionState(int controllerEpoch,
- int leader,
- int leaderEpoch,
- List isr,
- int zkVersion,
- List replicas,
- List addingReplicas,
- List removingReplicas,
- boolean isNew) {
- this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
- this.addingReplicas = addingReplicas;
- this.removingReplicas = removingReplicas;
- this.isNew = isNew;
- }
-
- private PartitionState(Struct struct) {
- int controllerEpoch = struct.get(CONTROLLER_EPOCH);
- int leader = struct.get(LEADER);
- int leaderEpoch = struct.get(LEADER_EPOCH);
-
- Object[] isrArray = struct.get(ISR);
- List isr = new ArrayList<>(isrArray.length);
- for (Object r : isrArray)
- isr.add((Integer) r);
-
- int zkVersion = struct.get(ZK_VERSION);
-
- Object[] replicasArray = struct.get(REPLICAS);
- List replicas = new ArrayList<>(replicasArray.length);
- for (Object r : replicasArray)
- replicas.add((Integer) r);
-
- this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
-
- List addingReplicas = new ArrayList<>();
- if (struct.hasField(ADDING_REPLICAS)) {
- for (Object r : struct.get(ADDING_REPLICAS))
- addingReplicas.add((Integer) r);
- }
- this.addingReplicas = addingReplicas;
-
- List removingReplicas = new ArrayList<>();
- if (struct.hasField(REMOVING_REPLICAS)) {
- for (Object r : struct.get(REMOVING_REPLICAS))
- removingReplicas.add((Integer) r);
- }
- this.removingReplicas = removingReplicas;
-
- this.isNew = struct.getOrElse(IS_NEW, false);
- }
-
- @Override
- public String toString() {
- return "PartitionState(controllerEpoch=" + basePartitionState.controllerEpoch +
- ", leader=" + basePartitionState.leader +
- ", leaderEpoch=" + basePartitionState.leaderEpoch +
- ", isr=" + Utils.join(basePartitionState.isr, ",") +
- ", zkVersion=" + basePartitionState.zkVersion +
- ", replicas=" + Utils.join(basePartitionState.replicas, ",") +
- ", addingReplicas=" + Utils.join(addingReplicas, ",") +
- ", removingReplicas=" + Utils.join(removingReplicas, ",") +
- ", isNew=" + isNew + ")";
- }
+ protected int size() {
+ return data.size(version());
+ }
- private void setStruct(Struct struct, short version) {
- struct.set(CONTROLLER_EPOCH, basePartitionState.controllerEpoch);
- struct.set(LEADER, basePartitionState.leader);
- struct.set(LEADER_EPOCH, basePartitionState.leaderEpoch);
- struct.set(ISR, basePartitionState.isr.toArray());
- struct.set(ZK_VERSION, basePartitionState.zkVersion);
- struct.set(REPLICAS, basePartitionState.replicas.toArray());
- if (version >= 3) {
- struct.set(ADDING_REPLICAS, addingReplicas.toArray());
- struct.set(REMOVING_REPLICAS, removingReplicas.toArray());
- }
- struct.setIfExists(IS_NEW, isNew);
- }
+ public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) {
+ return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version);
}
+
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
index 3b80222eff0e..0329307e6669 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
@@ -16,45 +16,19 @@
*/
package org.apache.kafka.common.requests;
-import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData;
+import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
+import java.util.stream.Collectors;
public class LeaderAndIsrResponse extends AbstractResponse {
- private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions", "Response for the requests partitions");
-
- private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
- TOPIC_NAME,
- PARTITION_ID,
- ERROR_CODE);
- private static final Schema LEADER_AND_ISR_RESPONSE_V0 = new Schema(
- ERROR_CODE,
- PARTITIONS_V0);
-
- // LeaderAndIsrResponse V1 may receive KAFKA_STORAGE_ERROR in the response
- private static final Schema LEADER_AND_ISR_RESPONSE_V1 = LEADER_AND_ISR_RESPONSE_V0;
-
- private static final Schema LEADER_AND_ISR_RESPONSE_V2 = LEADER_AND_ISR_RESPONSE_V1;
-
- private static final Schema LEADER_AND_ISR_RESPONSE_V3 = LEADER_AND_ISR_RESPONSE_V2;
-
- public static Schema[] schemaVersions() {
- return new Schema[]{LEADER_AND_ISR_RESPONSE_V0, LEADER_AND_ISR_RESPONSE_V1, LEADER_AND_ISR_RESPONSE_V2, LEADER_AND_ISR_RESPONSE_V3};
- }
/**
* Possible error code:
@@ -62,74 +36,45 @@ public static Schema[] schemaVersions() {
* STALE_CONTROLLER_EPOCH (11)
* STALE_BROKER_EPOCH (77)
*/
- private final Errors error;
-
- private final Map responses;
+ private final LeaderAndIsrResponseData data;
- public LeaderAndIsrResponse(Errors error, Map responses) {
- this.responses = responses;
- this.error = error;
+ public LeaderAndIsrResponse(LeaderAndIsrResponseData data) {
+ this.data = data;
}
- public LeaderAndIsrResponse(Struct struct) {
- responses = new HashMap<>();
- for (Object responseDataObj : struct.get(PARTITIONS)) {
- Struct responseData = (Struct) responseDataObj;
- String topic = responseData.get(TOPIC_NAME);
- int partition = responseData.get(PARTITION_ID);
- Errors error = Errors.forCode(responseData.get(ERROR_CODE));
- responses.put(new TopicPartition(topic, partition), error);
- }
-
- error = Errors.forCode(struct.get(ERROR_CODE));
+ public LeaderAndIsrResponse(Struct struct, short version) {
+ this.data = new LeaderAndIsrResponseData(struct, version);
}
- public Map responses() {
- return responses;
+ public List partitions() {
+ return data.partitionErrors();
}
public Errors error() {
- return error;
+ return Errors.forCode(data.errorCode());
}
@Override
public Map errorCounts() {
+ Errors error = error();
if (error != Errors.NONE)
// Minor optimization since the top-level error applies to all partitions
- return Collections.singletonMap(error, responses.size());
- return errorCounts(responses);
+ return Collections.singletonMap(error, data.partitionErrors().size());
+ return errorCounts(data.partitionErrors().stream().map(l -> Errors.forCode(l.errorCode())).collect(Collectors.toList()));
}
public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) {
- return new LeaderAndIsrResponse(ApiKeys.LEADER_AND_ISR.parseResponse(version, buffer));
+ return new LeaderAndIsrResponse(ApiKeys.LEADER_AND_ISR.parseResponse(version, buffer), version);
}
@Override
protected Struct toStruct(short version) {
- Struct struct = new Struct(ApiKeys.LEADER_AND_ISR.responseSchema(version));
-
- List responseDatas = new ArrayList<>(responses.size());
- for (Map.Entry response : responses.entrySet()) {
- Struct partitionData = struct.instance(PARTITIONS);
- TopicPartition partition = response.getKey();
- partitionData.set(TOPIC_NAME, partition.topic());
- partitionData.set(PARTITION_ID, partition.partition());
- partitionData.set(ERROR_CODE, response.getValue().code());
- responseDatas.add(partitionData);
- }
-
- struct.set(PARTITIONS, responseDatas.toArray());
- struct.set(ERROR_CODE, error.code());
-
- return struct;
+ return data.toStruct(version);
}
@Override
public String toString() {
- return "LeaderAndIsrResponse(" +
- "responses=" + responses +
- ", error=" + error +
- ")";
+ return data.toString();
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
index b0e8860f8c10..caa6f47906bf 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
@@ -68,24 +68,13 @@ public ListGroupsRequest(Struct struct, short version) {
@Override
public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- short versionId = version();
- switch (versionId) {
- case 0:
- return new ListGroupsResponse(new ListGroupsResponseData()
- .setGroups(Collections.emptyList())
- .setErrorCode(Errors.forException(e).code())
- );
- case 1:
- case 2:
- return new ListGroupsResponse(new ListGroupsResponseData()
- .setGroups(Collections.emptyList())
- .setErrorCode(Errors.forException(e).code())
- .setThrottleTimeMs(throttleTimeMs)
- );
- default:
- throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
- versionId, this.getClass().getSimpleName(), ApiKeys.LIST_GROUPS.latestVersion()));
+ ListGroupsResponseData listGroupsResponseData = new ListGroupsResponseData().
+ setGroups(Collections.emptyList()).
+ setErrorCode(Errors.forException(e).code());
+ if (version() >= 1) {
+ listGroupsResponseData.setThrottleTimeMs(throttleTimeMs);
}
+ return new ListGroupsResponse(listGroupsResponseData);
}
public static ListGroupsRequest parse(ByteBuffer buffer, short version) {
@@ -94,6 +83,6 @@ public static ListGroupsRequest parse(ByteBuffer buffer, short version) {
@Override
protected Struct toStruct() {
- return new Struct(ApiKeys.LIST_GROUPS.requestSchema(version()));
+ return data.toStruct(version());
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
index 9f30d3b911d0..3adeb157c188 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
@@ -27,6 +27,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -94,14 +95,13 @@ public OffsetCommitResponseData data() {
@Override
public Map errorCounts() {
- Map errorMap = new HashMap<>();
+ List errors = new ArrayList<>();
for (OffsetCommitResponseTopic topic : data.topics()) {
for (OffsetCommitResponsePartition partition : topic.partitions()) {
- errorMap.put(new TopicPartition(topic.name(), partition.partitionIndex()),
- Errors.forCode(partition.errorCode()));
+ errors.add(Errors.forCode(partition.errorCode()));
}
}
- return errorCounts(errorMap);
+ return errorCounts(errors);
}
public static OffsetCommitResponse parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java
new file mode 100644
index 000000000000..293efc2f6f2d
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.OffsetDeleteRequestData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+
+public class OffsetDeleteRequest extends AbstractRequest {
+
+ public static class Builder extends AbstractRequest.Builder {
+
+ private final OffsetDeleteRequestData data;
+
+ public Builder(OffsetDeleteRequestData data) {
+ super(ApiKeys.OFFSET_DELETE);
+ this.data = data;
+ }
+
+ @Override
+ public OffsetDeleteRequest build(short version) {
+ return new OffsetDeleteRequest(data, version);
+ }
+
+ @Override
+ public String toString() {
+ return data.toString();
+ }
+ }
+
+ public final OffsetDeleteRequestData data;
+
+ public OffsetDeleteRequest(OffsetDeleteRequestData data, short version) {
+ super(ApiKeys.OFFSET_DELETE, version);
+ this.data = data;
+ }
+
+ public OffsetDeleteRequest(Struct struct, short version) {
+ super(ApiKeys.OFFSET_DELETE, version);
+ this.data = new OffsetDeleteRequestData(struct, version);
+ }
+
+ public AbstractResponse getErrorResponse(int throttleTimeMs, Errors error) {
+ return new OffsetDeleteResponse(
+ new OffsetDeleteResponseData()
+ .setThrottleTimeMs(throttleTimeMs)
+ .setErrorCode(error.code())
+ );
+ }
+
+ @Override
+ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
+ return getErrorResponse(throttleTimeMs, Errors.forException(e));
+ }
+
+ public static OffsetDeleteRequest parse(ByteBuffer buffer, short version) {
+ return new OffsetDeleteRequest(ApiKeys.OFFSET_DELETE.parseRequest(version, buffer),
+ version);
+ }
+
+ @Override
+ protected Struct toStruct() {
+ return data.toStruct(version());
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java
new file mode 100644
index 000000000000..ff0c2967be89
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartition;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.Struct;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Possible error codes:
+ *
+ * - Partition errors:
+ * - {@link Errors#GROUP_SUBSCRIBED_TO_TOPIC}
+ * - {@link Errors#TOPIC_AUTHORIZATION_FAILED}
+ * - {@link Errors#UNKNOWN_TOPIC_OR_PARTITION}
+ *
+ * - Group or coordinator errors:
+ * - {@link Errors#COORDINATOR_LOAD_IN_PROGRESS}
+ * - {@link Errors#COORDINATOR_NOT_AVAILABLE}
+ * - {@link Errors#NOT_COORDINATOR}
+ * - {@link Errors#GROUP_AUTHORIZATION_FAILED}
+ * - {@link Errors#INVALID_GROUP_ID}
+ * - {@link Errors#GROUP_ID_NOT_FOUND}
+ * - {@link Errors#NON_EMPTY_GROUP}
+ */
+public class OffsetDeleteResponse extends AbstractResponse {
+
+ public final OffsetDeleteResponseData data;
+
+ public OffsetDeleteResponse(OffsetDeleteResponseData data) {
+ this.data = data;
+ }
+
+ public OffsetDeleteResponse(Struct struct) {
+ short latestVersion = (short) (OffsetDeleteResponseData.SCHEMAS.length - 1);
+ this.data = new OffsetDeleteResponseData(struct, latestVersion);
+ }
+
+ public OffsetDeleteResponse(Struct struct, short version) {
+ this.data = new OffsetDeleteResponseData(struct, version);
+ }
+
+ @Override
+ protected Struct toStruct(short version) {
+ return data.toStruct(version);
+ }
+
+ @Override
+ public Map errorCounts() {
+ Map counts = new HashMap<>();
+ counts.put(Errors.forCode(data.errorCode()), 1);
+ for (OffsetDeleteResponseTopic topic : data.topics()) {
+ for (OffsetDeleteResponsePartition partition : topic.partitions()) {
+ Errors error = Errors.forCode(partition.errorCode());
+ counts.put(error, counts.getOrDefault(error, 0) + 1);
+ }
+ }
+ return counts;
+ }
+
+ public static OffsetDeleteResponse parse(ByteBuffer buffer, short version) {
+ return new OffsetDeleteResponse(ApiKeys.OFFSET_DELETE.parseResponse(version, buffer));
+ }
+
+ @Override
+ public int throttleTimeMs() {
+ return data.throttleTimeMs();
+ }
+
+ @Override
+ public boolean shouldClientThrottle(short version) {
+ return version >= 0;
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
index f616ac1df810..2303f89f9198 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
@@ -118,10 +118,8 @@ public OffsetFetchResponse getErrorResponse(Errors error) {
}
public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Errors error) {
- short versionId = version();
-
Map responsePartitions = new HashMap<>();
- if (versionId < 2) {
+ if (version() < 2) {
OffsetFetchResponse.PartitionData partitionError = new OffsetFetchResponse.PartitionData(
OffsetFetchResponse.INVALID_OFFSET,
Optional.empty(),
@@ -136,18 +134,10 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Errors error) {
}
}
- switch (versionId) {
- case 0:
- case 1:
- case 2:
- return new OffsetFetchResponse(error, responsePartitions);
- case 3:
- case 4:
- case 5:
- return new OffsetFetchResponse(throttleTimeMs, error, responsePartitions);
- default:
- throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
- versionId, this.getClass().getSimpleName(), ApiKeys.OFFSET_FETCH.latestVersion()));
+ if (version() >= 3) {
+ return new OffsetFetchResponse(throttleTimeMs, error, responsePartitions);
+ } else {
+ return new OffsetFetchResponse(error, responsePartitions);
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
index 4002d2a5f44d..c067444478b0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java
@@ -91,7 +91,7 @@ public short apiVersion() {
}
@Override
- public String listener() {
+ public String listenerName() {
return listenerName.value();
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
index e5ec43acebe8..65bc39d230a2 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
@@ -17,123 +17,94 @@
package org.apache.kafka.common.requests;
import org.apache.kafka.common.errors.InvalidRequestException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.RequestHeaderData;
import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-import java.util.Objects;
-
-import static java.util.Objects.requireNonNull;
-import static org.apache.kafka.common.protocol.types.Type.INT16;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
-import static org.apache.kafka.common.protocol.types.Type.NULLABLE_STRING;
/**
* The header for a request in the Kafka protocol
*/
public class RequestHeader extends AbstractRequestResponse {
- private static final String API_KEY_FIELD_NAME = "api_key";
- private static final String API_VERSION_FIELD_NAME = "api_version";
- private static final String CLIENT_ID_FIELD_NAME = "client_id";
- private static final String CORRELATION_ID_FIELD_NAME = "correlation_id";
-
- public static final Schema SCHEMA = new Schema(
- new Field(API_KEY_FIELD_NAME, INT16, "The id of the request type."),
- new Field(API_VERSION_FIELD_NAME, INT16, "The version of the API."),
- new Field(CORRELATION_ID_FIELD_NAME, INT32, "A user-supplied integer value that will be passed back with the response"),
- new Field(CLIENT_ID_FIELD_NAME, NULLABLE_STRING, "A user specified identifier for the client making the request.", ""));
-
- // Version 0 of the controlled shutdown API used a non-standard request header (the clientId is missing).
- // This can be removed once we drop support for that version.
- private static final Schema CONTROLLED_SHUTDOWN_V0_SCHEMA = new Schema(
- new Field(API_KEY_FIELD_NAME, INT16, "The id of the request type."),
- new Field(API_VERSION_FIELD_NAME, INT16, "The version of the API."),
- new Field(CORRELATION_ID_FIELD_NAME, INT32, "A user-supplied integer value that will be passed back with the response"));
-
- private final ApiKeys apiKey;
- private final short apiVersion;
- private final String clientId;
- private final int correlationId;
-
- public RequestHeader(Struct struct) {
- short apiKey = struct.getShort(API_KEY_FIELD_NAME);
- if (!ApiKeys.hasId(apiKey))
- throw new InvalidRequestException("Unknown API key " + apiKey);
-
- this.apiKey = ApiKeys.forId(apiKey);
- apiVersion = struct.getShort(API_VERSION_FIELD_NAME);
-
- // only v0 of the controlled shutdown request is missing the clientId
- if (struct.hasField(CLIENT_ID_FIELD_NAME))
- clientId = struct.getString(CLIENT_ID_FIELD_NAME);
- else
- clientId = "";
- correlationId = struct.getInt(CORRELATION_ID_FIELD_NAME);
+ private final RequestHeaderData data;
+ private final short headerVersion;
+
+ public RequestHeader(Struct struct, short headerVersion) {
+ this(new RequestHeaderData(struct, headerVersion), headerVersion);
+ }
+
+ public RequestHeader(ApiKeys requestApiKey, short requestVersion, String clientId, int correlationId) {
+ this(new RequestHeaderData().
+ setRequestApiKey(requestApiKey.id).
+ setRequestApiVersion(requestVersion).
+ setClientId(clientId).
+ setCorrelationId(correlationId),
+ ApiKeys.forId(requestApiKey.id).headerVersion(requestVersion));
}
- public RequestHeader(ApiKeys apiKey, short version, String clientId, int correlation) {
- this.apiKey = requireNonNull(apiKey);
- this.apiVersion = version;
- this.clientId = clientId;
- this.correlationId = correlation;
+ public RequestHeader(RequestHeaderData data, short headerVersion) {
+ this.data = data;
+ this.headerVersion = headerVersion;
}
public Struct toStruct() {
- Schema schema = schema(apiKey.id, apiVersion);
- Struct struct = new Struct(schema);
- struct.set(API_KEY_FIELD_NAME, apiKey.id);
- struct.set(API_VERSION_FIELD_NAME, apiVersion);
-
- // only v0 of the controlled shutdown request is missing the clientId
- if (struct.hasField(CLIENT_ID_FIELD_NAME))
- struct.set(CLIENT_ID_FIELD_NAME, clientId);
- struct.set(CORRELATION_ID_FIELD_NAME, correlationId);
- return struct;
+ return this.data.toStruct(headerVersion);
}
public ApiKeys apiKey() {
- return apiKey;
+ return ApiKeys.forId(data.requestApiKey());
}
public short apiVersion() {
- return apiVersion;
+ return data.requestApiVersion();
+ }
+
+ public short headerVersion() {
+ return headerVersion;
}
public String clientId() {
- return clientId;
+ return data.clientId();
}
public int correlationId() {
- return correlationId;
+ return data.correlationId();
+ }
+
+ public RequestHeaderData data() {
+ return data;
}
public ResponseHeader toResponseHeader() {
- return new ResponseHeader(correlationId);
+ return new ResponseHeader(data.correlationId(), headerVersion);
}
public static RequestHeader parse(ByteBuffer buffer) {
+ short apiKey = -1;
try {
- short apiKey = buffer.getShort();
+ apiKey = buffer.getShort();
short apiVersion = buffer.getShort();
- Schema schema = schema(apiKey, apiVersion);
+ short headerVersion = ApiKeys.forId(apiKey).headerVersion(apiVersion);
buffer.rewind();
- return new RequestHeader(schema.read(buffer));
- } catch (InvalidRequestException e) {
- throw e;
- } catch (Throwable ex) {
+ return new RequestHeader(new RequestHeaderData(
+ new ByteBufferAccessor(buffer), headerVersion), headerVersion);
+ } catch (UnsupportedVersionException e) {
+ throw new InvalidRequestException("Unknown API key " + apiKey, e);
+ } catch (Throwable ex) {
throw new InvalidRequestException("Error parsing request header. Our best guess of the apiKey is: " +
- buffer.getShort(0), ex);
+ apiKey, ex);
}
}
@Override
public String toString() {
- return "RequestHeader(apiKey=" + apiKey +
- ", apiVersion=" + apiVersion +
- ", clientId=" + clientId +
- ", correlationId=" + correlationId +
+ return "RequestHeader(apiKey=" + apiKey() +
+ ", apiVersion=" + apiVersion() +
+ ", clientId=" + clientId() +
+ ", correlationId=" + correlationId() +
")";
}
@@ -141,28 +112,12 @@ public String toString() {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
RequestHeader that = (RequestHeader) o;
- return apiKey == that.apiKey &&
- apiVersion == that.apiVersion &&
- correlationId == that.correlationId &&
- Objects.equals(clientId, that.clientId);
+ return this.data.equals(that.data);
}
@Override
public int hashCode() {
- int result = apiKey.hashCode();
- result = 31 * result + (int) apiVersion;
- result = 31 * result + (clientId != null ? clientId.hashCode() : 0);
- result = 31 * result + correlationId;
- return result;
- }
-
- private static Schema schema(short apiKey, short version) {
- if (apiKey == ApiKeys.CONTROLLED_SHUTDOWN.id && version == 0)
- // This will be removed once we remove support for v0 of ControlledShutdownRequest, which
- // depends on a non-standard request header (it does not have a clientId)
- return CONTROLLED_SHUTDOWN_V0_SCHEMA;
- return SCHEMA;
+ return this.data.hashCode();
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
index fe452a2dc080..249b5d021081 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java
@@ -16,31 +16,30 @@
*/
package org.apache.kafka.common.requests;
-import org.apache.kafka.common.protocol.types.BoundField;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
+import org.apache.kafka.common.message.ResponseHeaderData;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
-
/**
* A response header in the kafka protocol.
*/
public class ResponseHeader extends AbstractRequestResponse {
- public static final Schema SCHEMA = new Schema(
- new Field("correlation_id", INT32, "The user-supplied value passed in with the request"));
- private static final BoundField CORRELATION_KEY_FIELD = SCHEMA.get("correlation_id");
+ private final ResponseHeaderData data;
+ private final short headerVersion;
- private final int correlationId;
+ public ResponseHeader(Struct struct, short headerVersion) {
+ this(new ResponseHeaderData(struct, headerVersion), headerVersion);
+ }
- public ResponseHeader(Struct struct) {
- correlationId = struct.getInt(CORRELATION_KEY_FIELD);
+ public ResponseHeader(int correlationId, short headerVersion) {
+ this(new ResponseHeaderData().setCorrelationId(correlationId), headerVersion);
}
- public ResponseHeader(int correlationId) {
- this.correlationId = correlationId;
+ public ResponseHeader(ResponseHeaderData data, short headerVersion) {
+ this.data = data;
+ this.headerVersion = headerVersion;
}
public int sizeOf() {
@@ -48,17 +47,24 @@ public int sizeOf() {
}
public Struct toStruct() {
- Struct struct = new Struct(SCHEMA);
- struct.set(CORRELATION_KEY_FIELD, correlationId);
- return struct;
+ return data.toStruct(headerVersion);
}
public int correlationId() {
- return correlationId;
+ return this.data.correlationId();
}
- public static ResponseHeader parse(ByteBuffer buffer) {
- return new ResponseHeader(SCHEMA.read(buffer));
+ public short headerVersion() {
+ return headerVersion;
}
+ public ResponseHeaderData data() {
+ return data;
+ }
+
+ public static ResponseHeader parse(ByteBuffer buffer, short headerVersion) {
+ return new ResponseHeader(
+ new ResponseHeaderData(new ByteBufferAccessor(buffer), headerVersion),
+ headerVersion);
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
index bad9573a1575..e79228917fb3 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
@@ -17,57 +17,27 @@
package org.apache.kafka.common.requests;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.StopReplicaRequestData;
+import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaPartitionV0;
+import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaTopic;
+import org.apache.kafka.common.message.StopReplicaResponseData;
+import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
import org.apache.kafka.common.utils.CollectionUtils;
+import org.apache.kafka.common.utils.FlattenedIterator;
+import org.apache.kafka.common.utils.MappedIterator;
import org.apache.kafka.common.utils.Utils;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
+import java.util.stream.Collectors;
public class StopReplicaRequest extends AbstractControlRequest {
- private static final Field.Bool DELETE_PARTITIONS = new Field.Bool("delete_partitions", "Boolean which indicates if replica's partitions must be deleted.");
- private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions", "The partitions");
- private static final Field.Array PARTITION_IDS = new Field.Array("partition_ids", INT32, "The partition ids of a topic");
-
- private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
- TOPIC_NAME,
- PARTITION_ID);
- private static final Field PARTITIONS_V1 = PARTITIONS.withFields(
- TOPIC_NAME,
- PARTITION_IDS);
-
- private static final Schema STOP_REPLICA_REQUEST_V0 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- DELETE_PARTITIONS,
- PARTITIONS_V0);
-
- // STOP_REPLICA_REQUEST_V1 added a broker_epoch Field. This field specifies the generation of the broker across
- // bounces. It also normalizes partitions under each topic.
- private static final Schema STOP_REPLICA_REQUEST_V1 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- BROKER_EPOCH,
- DELETE_PARTITIONS,
- PARTITIONS_V1);
-
-
- public static Schema[] schemaVersions() {
- return new Schema[] {STOP_REPLICA_REQUEST_V0, STOP_REPLICA_REQUEST_V1};
- }
public static class Builder extends AbstractControlRequest.Builder {
private final boolean deletePartitions;
@@ -80,10 +50,31 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
this.partitions = partitions;
}
- @Override
public StopReplicaRequest build(short version) {
- return new StopReplicaRequest(controllerId, controllerEpoch, brokerEpoch,
- deletePartitions, partitions, version);
+ StopReplicaRequestData data = new StopReplicaRequestData()
+ .setControllerId(controllerId)
+ .setControllerEpoch(controllerEpoch)
+ .setBrokerEpoch(brokerEpoch)
+ .setDeletePartitions(deletePartitions);
+
+ if (version >= 1) {
+ Map> topicPartitionsMap = CollectionUtils.groupPartitionsByTopic(partitions);
+ List topics = topicPartitionsMap.entrySet().stream().map(entry ->
+ new StopReplicaTopic()
+ .setName(entry.getKey())
+ .setPartitionIndexes(entry.getValue())
+ ).collect(Collectors.toList());
+ data.setTopics(topics);
+ } else {
+ List requestPartitions = partitions.stream().map(tp ->
+ new StopReplicaPartitionV0()
+ .setTopicName(tp.topic())
+ .setPartitionIndex(tp.partition())
+ ).collect(Collectors.toList());
+ data.setUngroupedPartitions(requestPartitions);
+ }
+
+ return new StopReplicaRequest(data, version);
}
@Override
@@ -100,54 +91,37 @@ public String toString() {
}
}
- private final boolean deletePartitions;
- private final Collection partitions;
+ private final StopReplicaRequestData data;
- private StopReplicaRequest(int controllerId, int controllerEpoch, long brokerEpoch, boolean deletePartitions,
- Collection partitions, short version) {
- super(ApiKeys.STOP_REPLICA, version, controllerId, controllerEpoch, brokerEpoch);
- this.deletePartitions = deletePartitions;
- this.partitions = partitions;
+ private StopReplicaRequest(StopReplicaRequestData data, short version) {
+ super(ApiKeys.STOP_REPLICA, version);
+ this.data = data;
}
public StopReplicaRequest(Struct struct, short version) {
- super(ApiKeys.STOP_REPLICA, struct, version);
-
- partitions = new HashSet<>();
- if (version > 0) { // V1
- for (Object topicObj : struct.get(PARTITIONS)) {
- Struct topicData = (Struct) topicObj;
- String topic = topicData.get(TOPIC_NAME);
- for (Object partitionObj : topicData.get(PARTITION_IDS)) {
- int partition = (Integer) partitionObj;
- partitions.add(new TopicPartition(topic, partition));
- }
- }
- } else { // V0
- for (Object partitionDataObj : struct.get(PARTITIONS)) {
- Struct partitionData = (Struct) partitionDataObj;
- String topic = partitionData.get(TOPIC_NAME);
- int partition = partitionData.get(PARTITION_ID);
- partitions.add(new TopicPartition(topic, partition));
- }
- }
- deletePartitions = struct.get(DELETE_PARTITIONS);
+ this(new StopReplicaRequestData(struct, version), version);
}
@Override
public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) {
Errors error = Errors.forException(e);
- Map responses = new HashMap<>(partitions.size());
- for (TopicPartition partition : partitions) {
- responses.put(partition, error);
+ StopReplicaResponseData data = new StopReplicaResponseData();
+ data.setErrorCode(error.code());
+ List partitions = new ArrayList<>();
+ for (TopicPartition tp : partitions()) {
+ partitions.add(new StopReplicaPartitionError()
+ .setTopicName(tp.topic())
+ .setPartitionIndex(tp.partition())
+ .setErrorCode(error.code()));
}
+ data.setPartitionErrors(partitions);
short versionId = version();
switch (versionId) {
case 0:
case 1:
- return new StopReplicaResponse(error, responses);
+ return new StopReplicaResponse(data);
default:
throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
versionId, this.getClass().getSimpleName(), ApiKeys.STOP_REPLICA.latestVersion()));
@@ -155,11 +129,40 @@ public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) {
}
public boolean deletePartitions() {
- return deletePartitions;
+ return data.deletePartitions();
+ }
+
+ /**
+ * Note that this method has allocation overhead per iterated element, so callers should copy the result into
+ * another collection if they need to iterate more than once.
+ *
+ * Implementation note: we should strive to avoid allocation overhead per element, see
+ * `UpdateMetadataRequest.partitionStates()` for the preferred approach. That's not possible in this case and
+ * StopReplicaRequest should be relatively rare in comparison to other request types.
+ */
+ public Iterable partitions() {
+ if (version() >= 1) {
+ return () -> new FlattenedIterator<>(data.topics().iterator(), topic ->
+ new MappedIterator<>(topic.partitionIndexes().iterator(), partition ->
+ new TopicPartition(topic.name(), partition)));
+ }
+ return () -> new MappedIterator<>(data.ungroupedPartitions().iterator(),
+ partition -> new TopicPartition(partition.topicName(), partition.partitionIndex()));
+ }
+
+ @Override
+ public int controllerId() {
+ return data.controllerId();
+ }
+
+ @Override
+ public int controllerEpoch() {
+ return data.controllerEpoch();
}
- public Collection partitions() {
- return partitions;
+ @Override
+ public long brokerEpoch() {
+ return data.brokerEpoch();
}
public static StopReplicaRequest parse(ByteBuffer buffer, short version) {
@@ -168,34 +171,11 @@ public static StopReplicaRequest parse(ByteBuffer buffer, short version) {
@Override
protected Struct toStruct() {
- Struct struct = new Struct(ApiKeys.STOP_REPLICA.requestSchema(version()));
-
- struct.set(CONTROLLER_ID, controllerId);
- struct.set(CONTROLLER_EPOCH, controllerEpoch);
- struct.setIfExists(BROKER_EPOCH, brokerEpoch);
- struct.set(DELETE_PARTITIONS, deletePartitions);
-
- if (version() > 0) { // V1
- Map> topicPartitionsMap = CollectionUtils.groupPartitionsByTopic(partitions);
- List topicsData = new ArrayList<>(topicPartitionsMap.size());
- for (Map.Entry> entry : topicPartitionsMap.entrySet()) {
- Struct topicData = struct.instance(PARTITIONS);
- topicData.set(TOPIC_NAME, entry.getKey());
- topicData.set(PARTITION_IDS, entry.getValue().toArray());
- topicsData.add(topicData);
- }
- struct.set(PARTITIONS, topicsData.toArray());
-
- } else { // V0
- List partitionDatas = new ArrayList<>(partitions.size());
- for (TopicPartition partition : partitions) {
- Struct partitionData = struct.instance(PARTITIONS);
- partitionData.set(TOPIC_NAME, partition.topic());
- partitionData.set(PARTITION_ID, partition.partition());
- partitionDatas.add(partitionData);
- }
- struct.set(PARTITIONS, partitionDatas.toArray());
- }
- return struct;
+ return data.toStruct(version());
}
+
+ protected long size() {
+ return data.size(version());
+ }
+
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
index 6089bea8d4f0..7d6c7a074289 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
@@ -16,43 +16,19 @@
*/
package org.apache.kafka.common.requests;
-import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.StopReplicaResponseData;
+import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
+import java.util.stream.Collectors;
public class StopReplicaResponse extends AbstractResponse {
- private static final Field.ComplexArray PARTITIONS = new Field.ComplexArray("partitions", "Response for the requests partitions");
-
- private static final Field PARTITIONS_V0 = PARTITIONS.withFields(
- TOPIC_NAME,
- PARTITION_ID,
- ERROR_CODE);
- private static final Schema STOP_REPLICA_RESPONSE_V0 = new Schema(
- ERROR_CODE,
- PARTITIONS_V0);
-
- private static final Schema STOP_REPLICA_RESPONSE_V1 = STOP_REPLICA_RESPONSE_V0;
-
-
- public static Schema[] schemaVersions() {
- return new Schema[] {STOP_REPLICA_RESPONSE_V0, STOP_REPLICA_RESPONSE_V1};
- }
-
- private final Map responses;
/**
* Possible error code:
@@ -60,71 +36,44 @@ public static Schema[] schemaVersions() {
* STALE_CONTROLLER_EPOCH (11)
* STALE_BROKER_EPOCH (77)
*/
- private final Errors error;
+ private final StopReplicaResponseData data;
- public StopReplicaResponse(Errors error, Map responses) {
- this.responses = responses;
- this.error = error;
+ public StopReplicaResponse(StopReplicaResponseData data) {
+ this.data = data;
}
- public StopReplicaResponse(Struct struct) {
- responses = new HashMap<>();
- for (Object responseDataObj : struct.get(PARTITIONS)) {
- Struct responseData = (Struct) responseDataObj;
- String topic = responseData.get(TOPIC_NAME);
- int partition = responseData.get(PARTITION_ID);
- Errors error = Errors.forCode(responseData.get(ERROR_CODE));
- responses.put(new TopicPartition(topic, partition), error);
- }
-
- error = Errors.forCode(struct.get(ERROR_CODE));
+ public StopReplicaResponse(Struct struct, short version) {
+ data = new StopReplicaResponseData(struct, version);
}
- public Map responses() {
- return responses;
+ public List partitionErrors() {
+ return data.partitionErrors();
}
public Errors error() {
- return error;
+ return Errors.forCode(data.errorCode());
}
@Override
public Map errorCounts() {
- if (error != Errors.NONE)
+ if (data.errorCode() != Errors.NONE.code())
// Minor optimization since the top-level error applies to all partitions
- return Collections.singletonMap(error, responses.size());
- return errorCounts(responses);
+ return Collections.singletonMap(error(), data.partitionErrors().size());
+ return errorCounts(data.partitionErrors().stream().map(p -> Errors.forCode(p.errorCode())).collect(Collectors.toList()));
}
public static StopReplicaResponse parse(ByteBuffer buffer, short version) {
- return new StopReplicaResponse(ApiKeys.STOP_REPLICA.parseResponse(version, buffer));
+ return new StopReplicaResponse(ApiKeys.STOP_REPLICA.parseResponse(version, buffer), version);
}
@Override
protected Struct toStruct(short version) {
- Struct struct = new Struct(ApiKeys.STOP_REPLICA.responseSchema(version));
-
- List responseDatas = new ArrayList<>(responses.size());
- for (Map.Entry response : responses.entrySet()) {
- Struct partitionData = struct.instance(PARTITIONS);
- TopicPartition partition = response.getKey();
- partitionData.set(TOPIC_NAME, partition.topic());
- partitionData.set(PARTITION_ID, partition.partition());
- partitionData.set(ERROR_CODE, response.getValue().code());
- responseDatas.add(partitionData);
- }
-
- struct.set(PARTITIONS, responseDatas.toArray());
- struct.set(ERROR_CODE, error.code());
- return struct;
+ return data.toStruct(version);
}
@Override
public String toString() {
- return "StopReplicaResponse(" +
- "responses=" + responses +
- ", error=" + error +
- ")";
+ return data.toString();
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
index 1851f89f831d..256b9983671e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java
@@ -88,7 +88,7 @@ public int throttleTimeMs() {
@Override
public Map errorCounts() {
- return errorCounts(errors());
+ return errorCounts(errors().values());
}
public Map errors() {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index b715c2aa65b6..b73fa5b5c1a4 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -16,177 +16,38 @@
*/
package org.apache.kafka.common.requests;
-import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.UpdateMetadataRequestData;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
+import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataTopicState;
+import org.apache.kafka.common.message.UpdateMetadataResponseData;
import org.apache.kafka.common.network.ListenerName;
import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ByteBufferAccessor;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Field;
-import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
import org.apache.kafka.common.security.auth.SecurityProtocol;
-import org.apache.kafka.common.utils.CollectionUtils;
+import org.apache.kafka.common.utils.FlattenedIterator;
import org.apache.kafka.common.utils.Utils;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID;
-import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME;
-import static org.apache.kafka.common.protocol.types.Type.INT32;
+import static java.util.Collections.singletonList;
public class UpdateMetadataRequest extends AbstractControlRequest {
- private static final Field.ComplexArray TOPIC_STATES = new Field.ComplexArray("topic_states", "Topic states");
- private static final Field.ComplexArray PARTITION_STATES = new Field.ComplexArray("partition_states", "Partition states");
- private static final Field.ComplexArray LIVE_BROKERS = new Field.ComplexArray("live_brokers", "Live broekrs");
-
- // PartitionState fields
- private static final Field.Int32 LEADER = new Field.Int32("leader", "The broker id for the leader.");
- private static final Field.Int32 LEADER_EPOCH = new Field.Int32("leader_epoch", "The leader epoch.");
- private static final Field.Array ISR = new Field.Array("isr", INT32, "The in sync replica ids.");
- private static final Field.Int32 ZK_VERSION = new Field.Int32("zk_version", "The ZK version.");
- private static final Field.Array REPLICAS = new Field.Array("replicas", INT32, "The replica ids.");
- private static final Field.Array OFFLINE_REPLICAS = new Field.Array("offline_replicas", INT32, "The offline replica ids");
-
- // Live brokers fields
- private static final Field.Int32 BROKER_ID = new Field.Int32("id", "The broker id");
- private static final Field.ComplexArray ENDPOINTS = new Field.ComplexArray("end_points", "The endpoints");
- private static final Field.NullableStr RACK = new Field.NullableStr("rack", "The rack");
-
- // EndPoint fields
- private static final Field.Str HOST = new Field.Str("host", "The hostname of the broker.");
- private static final Field.Int32 PORT = new Field.Int32("port", "The port on which the broker accepts requests.");
- private static final Field.Str LISTENER_NAME = new Field.Str("listener_name", "The listener name.");
- private static final Field.Int16 SECURITY_PROTOCOL_TYPE = new Field.Int16("security_protocol_type", "The security protocol type.");
-
- private static final Field PARTITION_STATES_V0 = PARTITION_STATES.withFields(
- TOPIC_NAME,
- PARTITION_ID,
- CONTROLLER_EPOCH,
- LEADER,
- LEADER_EPOCH,
- ISR,
- ZK_VERSION,
- REPLICAS);
-
- // PARTITION_STATES_V4 added a per-partition offline_replicas field. This field specifies
- // the list of replicas that are offline.
- private static final Field PARTITION_STATES_V4 = PARTITION_STATES.withFields(
- TOPIC_NAME,
- PARTITION_ID,
- CONTROLLER_EPOCH,
- LEADER,
- LEADER_EPOCH,
- ISR,
- ZK_VERSION,
- REPLICAS,
- OFFLINE_REPLICAS);
-
- private static final Field PARTITION_STATES_V5 = PARTITION_STATES.withFields(
- PARTITION_ID,
- CONTROLLER_EPOCH,
- LEADER,
- LEADER_EPOCH,
- ISR,
- ZK_VERSION,
- REPLICAS,
- OFFLINE_REPLICAS);
-
- // TOPIC_STATES_V5 normalizes TOPIC_STATES_V4 to
- // make it more memory efficient
- private static final Field TOPIC_STATES_V5 = TOPIC_STATES.withFields(
- TOPIC_NAME,
- PARTITION_STATES_V5);
-
- // for some reason, V1 sends `port` before `host` while V0 sends `host` before `port
- private static final Field ENDPOINTS_V1 = ENDPOINTS.withFields(
- PORT,
- HOST,
- SECURITY_PROTOCOL_TYPE);
-
- private static final Field ENDPOINTS_V3 = ENDPOINTS.withFields(
- PORT,
- HOST,
- LISTENER_NAME,
- SECURITY_PROTOCOL_TYPE);
-
- private static final Field LIVE_BROKERS_V0 = LIVE_BROKERS.withFields(
- BROKER_ID,
- HOST,
- PORT);
-
- private static final Field LIVE_BROKERS_V1 = LIVE_BROKERS.withFields(
- BROKER_ID,
- ENDPOINTS_V1);
-
- private static final Field LIVE_BROKERS_V2 = LIVE_BROKERS.withFields(
- BROKER_ID,
- ENDPOINTS_V1,
- RACK);
-
- private static final Field LIVE_BROKERS_V3 = LIVE_BROKERS.withFields(
- BROKER_ID,
- ENDPOINTS_V3,
- RACK);
-
- private static final Schema UPDATE_METADATA_REQUEST_V0 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- PARTITION_STATES_V0,
- LIVE_BROKERS_V0);
-
- private static final Schema UPDATE_METADATA_REQUEST_V1 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- PARTITION_STATES_V0,
- LIVE_BROKERS_V1);
-
- private static final Schema UPDATE_METADATA_REQUEST_V2 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- PARTITION_STATES_V0,
- LIVE_BROKERS_V2);
-
-
- private static final Schema UPDATE_METADATA_REQUEST_V3 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- PARTITION_STATES_V0,
- LIVE_BROKERS_V3);
-
- // UPDATE_METADATA_REQUEST_V4 added a per-partition offline_replicas field. This field specifies the list of replicas that are offline.
- private static final Schema UPDATE_METADATA_REQUEST_V4 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- PARTITION_STATES_V4,
- LIVE_BROKERS_V3);
-
- // UPDATE_METADATA_REQUEST_V5 added a broker_epoch Field. This field specifies the generation of the broker across
- // bounces. It also normalizes partitions under each topic.
- private static final Schema UPDATE_METADATA_REQUEST_V5 = new Schema(
- CONTROLLER_ID,
- CONTROLLER_EPOCH,
- BROKER_EPOCH,
- TOPIC_STATES_V5,
- LIVE_BROKERS_V3);
-
- public static Schema[] schemaVersions() {
- return new Schema[] {UPDATE_METADATA_REQUEST_V0, UPDATE_METADATA_REQUEST_V1, UPDATE_METADATA_REQUEST_V2,
- UPDATE_METADATA_REQUEST_V3, UPDATE_METADATA_REQUEST_V4, UPDATE_METADATA_REQUEST_V5};
- }
public static class Builder extends AbstractControlRequest.Builder {
- private final Map partitionStates;
- private final Set liveBrokers;
+ private final List partitionStates;
+ private final List liveBrokers;
public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- Map partitionStates, Set liveBrokers) {
+ List partitionStates, List liveBrokers) {
super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch);
this.partitionStates = partitionStates;
this.liveBrokers = liveBrokers;
@@ -194,14 +55,53 @@ public Builder(short version, int controllerId, int controllerEpoch, long broker
@Override
public UpdateMetadataRequest build(short version) {
- if (version == 0) {
- for (Broker broker : liveBrokers) {
- if (broker.endPoints.size() != 1 || broker.endPoints.get(0).securityProtocol != SecurityProtocol.PLAINTEXT) {
- throw new UnsupportedVersionException("UpdateMetadataRequest v0 only handles PLAINTEXT endpoints");
+ if (version < 3) {
+ for (UpdateMetadataBroker broker : liveBrokers) {
+ if (version == 0) {
+ if (broker.endpoints().size() != 1)
+ throw new UnsupportedVersionException("UpdateMetadataRequest v0 requires a single endpoint");
+ if (broker.endpoints().get(0).securityProtocol() != SecurityProtocol.PLAINTEXT.id)
+ throw new UnsupportedVersionException("UpdateMetadataRequest v0 only handles PLAINTEXT endpoints");
+ // Don't null out `endpoints` since it's ignored by the generated code if version >= 1
+ UpdateMetadataEndpoint endpoint = broker.endpoints().get(0);
+ broker.setV0Host(endpoint.host());
+ broker.setV0Port(endpoint.port());
+ } else {
+ if (broker.endpoints().stream().anyMatch(endpoint -> !endpoint.listener().isEmpty() &&
+ !endpoint.listener().equals(listenerNameFromSecurityProtocol(endpoint)))) {
+ throw new UnsupportedVersionException("UpdateMetadataRequest v0-v3 does not support custom " +
+ "listeners, request version: " + version + ", endpoints: " + broker.endpoints());
+ }
}
}
}
- return new UpdateMetadataRequest(version, controllerId, controllerEpoch, brokerEpoch, partitionStates, liveBrokers);
+
+ UpdateMetadataRequestData data = new UpdateMetadataRequestData()
+ .setControllerId(controllerId)
+ .setControllerEpoch(controllerEpoch)
+ .setBrokerEpoch(brokerEpoch)
+ .setLiveBrokers(liveBrokers);
+
+ if (version >= 5) {
+ Map topicStatesMap = groupByTopic(partitionStates);
+ data.setTopicStates(new ArrayList<>(topicStatesMap.values()));
+ } else {
+ data.setUngroupedPartitionStates(partitionStates);
+ }
+
+ return new UpdateMetadataRequest(data, version);
+ }
+
+ private static Map groupByTopic(List partitionStates) {
+ Map topicStates = new HashMap<>();
+ for (UpdateMetadataPartitionState partition : partitionStates) {
+ // We don't null out the topic name in UpdateMetadataTopicState since it's ignored by the generated
+ // code if version >= 5
+ UpdateMetadataTopicState topicState = topicStates.computeIfAbsent(partition.topicName(),
+ t -> new UpdateMetadataTopicState().setTopicName(partition.topicName()));
+ topicState.partitionStates().add(partition);
+ }
+ return topicStates;
}
@Override
@@ -218,275 +118,110 @@ public String toString() {
}
}
- public static final class PartitionState {
- public final BasePartitionState basePartitionState;
- public final List offlineReplicas;
-
- public PartitionState(int controllerEpoch,
- int leader,
- int leaderEpoch,
- List isr,
- int zkVersion,
- List replicas,
- List offlineReplicas) {
- this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
- this.offlineReplicas = offlineReplicas;
- }
-
- private PartitionState(Struct struct) {
- int controllerEpoch = struct.get(CONTROLLER_EPOCH);
- int leader = struct.get(LEADER);
- int leaderEpoch = struct.get(LEADER_EPOCH);
-
- Object[] isrArray = struct.get(ISR);
- List isr = new ArrayList<>(isrArray.length);
- for (Object r : isrArray)
- isr.add((Integer) r);
-
- int zkVersion = struct.get(ZK_VERSION);
-
- Object[] replicasArray = struct.get(REPLICAS);
- List replicas = new ArrayList<>(replicasArray.length);
- for (Object r : replicasArray)
- replicas.add((Integer) r);
+ private final UpdateMetadataRequestData data;
- this.basePartitionState = new BasePartitionState(controllerEpoch, leader, leaderEpoch, isr, zkVersion, replicas);
+ UpdateMetadataRequest(UpdateMetadataRequestData data, short version) {
+ super(ApiKeys.UPDATE_METADATA, version);
+ this.data = data;
+ // Do this from the constructor to make it thread-safe (even though it's only needed when some methods are called)
+ normalize();
+ }
- this.offlineReplicas = new ArrayList<>();
- if (struct.hasField(OFFLINE_REPLICAS)) {
- Object[] offlineReplicasArray = struct.get(OFFLINE_REPLICAS);
- for (Object r : offlineReplicasArray)
- offlineReplicas.add((Integer) r);
+ private void normalize() {
+ // Version 0 only supported a single host and port and the protocol was always plaintext
+ // Version 1 added support for multiple endpoints, each with its own security protocol
+ // Version 2 added support for rack
+ // Version 3 added support for listener name, which we can infer from the security protocol for older versions
+ if (version() < 3) {
+ for (UpdateMetadataBroker liveBroker : data.liveBrokers()) {
+ // Set endpoints so that callers can rely on it always being present
+ if (version() == 0 && liveBroker.endpoints().isEmpty()) {
+ SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT;
+ liveBroker.setEndpoints(singletonList(new UpdateMetadataEndpoint()
+ .setHost(liveBroker.v0Host())
+ .setPort(liveBroker.v0Port())
+ .setSecurityProtocol(securityProtocol.id)
+ .setListener(ListenerName.forSecurityProtocol(securityProtocol).value())));
+ } else {
+ for (UpdateMetadataEndpoint endpoint : liveBroker.endpoints()) {
+ // Set listener so that callers can rely on it always being present
+ if (endpoint.listener().isEmpty())
+ endpoint.setListener(listenerNameFromSecurityProtocol(endpoint));
+ }
+ }
}
}
- @Override
- public String toString() {
- return "PartitionState(controllerEpoch=" + basePartitionState.controllerEpoch +
- ", leader=" + basePartitionState.leader +
- ", leaderEpoch=" + basePartitionState.leaderEpoch +
- ", isr=" + Arrays.toString(basePartitionState.isr.toArray()) +
- ", zkVersion=" + basePartitionState.zkVersion +
- ", replicas=" + Arrays.toString(basePartitionState.replicas.toArray()) +
- ", offlineReplicas=" + Arrays.toString(offlineReplicas.toArray()) + ")";
- }
-
- private void setStruct(Struct struct) {
- struct.set(CONTROLLER_EPOCH, basePartitionState.controllerEpoch);
- struct.set(LEADER, basePartitionState.leader);
- struct.set(LEADER_EPOCH, basePartitionState.leaderEpoch);
- struct.set(ISR, basePartitionState.isr.toArray());
- struct.set(ZK_VERSION, basePartitionState.zkVersion);
- struct.set(REPLICAS, basePartitionState.replicas.toArray());
- struct.setIfExists(OFFLINE_REPLICAS, offlineReplicas.toArray());
+ if (version() >= 5) {
+ for (UpdateMetadataTopicState topicState : data.topicStates()) {
+ for (UpdateMetadataPartitionState partitionState : topicState.partitionStates()) {
+ // Set the topic name so that we can always present the ungrouped view to callers
+ partitionState.setTopicName(topicState.topicName());
+ }
+ }
}
}
- public static final class Broker {
- public final int id;
- public final List endPoints;
- public final String rack; // introduced in V2
-
- public Broker(int id, List endPoints, String rack) {
- this.id = id;
- this.endPoints = endPoints;
- this.rack = rack;
- }
-
- @Override
- public String toString() {
- StringBuilder bld = new StringBuilder();
- bld.append("(id=").append(id);
- bld.append(", endPoints=").append(Utils.join(endPoints, ","));
- bld.append(", rack=").append(rack);
- bld.append(")");
- return bld.toString();
- }
+ private static String listenerNameFromSecurityProtocol(UpdateMetadataEndpoint endpoint) {
+ SecurityProtocol securityProtocol = SecurityProtocol.forId(endpoint.securityProtocol());
+ return ListenerName.forSecurityProtocol(securityProtocol).value();
}
- public static final class EndPoint {
- public final String host;
- public final int port;
- public final SecurityProtocol securityProtocol;
- public final ListenerName listenerName; // introduced in V3
-
- public EndPoint(String host, int port, SecurityProtocol securityProtocol, ListenerName listenerName) {
- this.host = host;
- this.port = port;
- this.securityProtocol = securityProtocol;
- this.listenerName = listenerName;
- }
-
- @Override
- public String toString() {
- return "(host=" + host + ", port=" + port + ", listenerName=" + listenerName +
- ", securityProtocol=" + securityProtocol + ")";
- }
+ public UpdateMetadataRequest(Struct struct, short version) {
+ this(new UpdateMetadataRequestData(struct, version), version);
}
- private final Map partitionStates;
- private final Set liveBrokers;
-
- private UpdateMetadataRequest(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- Map partitionStates, Set liveBrokers) {
- super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch);
- this.partitionStates = partitionStates;
- this.liveBrokers = liveBrokers;
+ @Override
+ public int controllerId() {
+ return data.controllerId();
}
- public UpdateMetadataRequest(Struct struct, short versionId) {
- super(ApiKeys.UPDATE_METADATA, struct, versionId);
- Map partitionStates = new HashMap<>();
- if (struct.hasField(TOPIC_STATES)) {
- for (Object topicStatesDataObj : struct.get(TOPIC_STATES)) {
- Struct topicStatesData = (Struct) topicStatesDataObj;
- String topic = topicStatesData.get(TOPIC_NAME);
- for (Object partitionStateDataObj : topicStatesData.get(PARTITION_STATES)) {
- Struct partitionStateData = (Struct) partitionStateDataObj;
- int partition = partitionStateData.get(PARTITION_ID);
- PartitionState partitionState = new PartitionState(partitionStateData);
- partitionStates.put(new TopicPartition(topic, partition), partitionState);
- }
- }
- } else {
- for (Object partitionStateDataObj : struct.get(PARTITION_STATES)) {
- Struct partitionStateData = (Struct) partitionStateDataObj;
- String topic = partitionStateData.get(TOPIC_NAME);
- int partition = partitionStateData.get(PARTITION_ID);
- PartitionState partitionState = new PartitionState(partitionStateData);
- partitionStates.put(new TopicPartition(topic, partition), partitionState);
- }
- }
-
- Set liveBrokers = new HashSet<>();
-
- for (Object brokerDataObj : struct.get(LIVE_BROKERS)) {
- Struct brokerData = (Struct) brokerDataObj;
- int brokerId = brokerData.get(BROKER_ID);
+ @Override
+ public int controllerEpoch() {
+ return data.controllerEpoch();
+ }
- // V0
- if (brokerData.hasField(HOST)) {
- String host = brokerData.get(HOST);
- int port = brokerData.get(PORT);
- List endPoints = new ArrayList<>(1);
- SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT;
- endPoints.add(new EndPoint(host, port, securityProtocol, ListenerName.forSecurityProtocol(securityProtocol)));
- liveBrokers.add(new Broker(brokerId, endPoints, null));
- } else { // V1, V2 or V3
- List endPoints = new ArrayList<>();
- for (Object endPointDataObj : brokerData.get(ENDPOINTS)) {
- Struct endPointData = (Struct) endPointDataObj;
- int port = endPointData.get(PORT);
- String host = endPointData.get(HOST);
- short protocolTypeId = endPointData.get(SECURITY_PROTOCOL_TYPE);
- SecurityProtocol securityProtocol = SecurityProtocol.forId(protocolTypeId);
- String listenerName;
- if (endPointData.hasField(LISTENER_NAME)) // V3
- listenerName = endPointData.get(LISTENER_NAME);
- else
- listenerName = securityProtocol.name;
- endPoints.add(new EndPoint(host, port, securityProtocol, new ListenerName(listenerName)));
- }
- String rack = null;
- if (brokerData.hasField(RACK)) { // V2
- rack = brokerData.get(RACK);
- }
- liveBrokers.add(new Broker(brokerId, endPoints, rack));
- }
- }
- this.partitionStates = partitionStates;
- this.liveBrokers = liveBrokers;
+ @Override
+ public long brokerEpoch() {
+ return data.brokerEpoch();
}
@Override
- protected Struct toStruct() {
+ public UpdateMetadataResponse getErrorResponse(int throttleTimeMs, Throwable e) {
short version = version();
- Struct struct = new Struct(ApiKeys.UPDATE_METADATA.requestSchema(version));
- struct.set(CONTROLLER_ID, controllerId);
- struct.set(CONTROLLER_EPOCH, controllerEpoch);
- struct.setIfExists(BROKER_EPOCH, brokerEpoch);
-
- if (struct.hasField(TOPIC_STATES)) {
- Map> topicStates = CollectionUtils.groupPartitionDataByTopic(partitionStates);
- List topicStatesData = new ArrayList<>(topicStates.size());
- for (Map.Entry> entry : topicStates.entrySet()) {
- Struct topicStateData = struct.instance(TOPIC_STATES);
- topicStateData.set(TOPIC_NAME, entry.getKey());
- Map partitionMap = entry.getValue();
- List partitionStatesData = new ArrayList<>(partitionMap.size());
- for (Map.Entry partitionEntry : partitionMap.entrySet()) {
- Struct partitionStateData = topicStateData.instance(PARTITION_STATES);
- partitionStateData.set(PARTITION_ID, partitionEntry.getKey());
- partitionEntry.getValue().setStruct(partitionStateData);
- partitionStatesData.add(partitionStateData);
- }
- topicStateData.set(PARTITION_STATES, partitionStatesData.toArray());
- topicStatesData.add(topicStateData);
- }
- struct.set(TOPIC_STATES, topicStatesData.toArray());
- } else {
- List partitionStatesData = new ArrayList<>(partitionStates.size());
- for (Map.Entry entry : partitionStates.entrySet()) {
- Struct partitionStateData = struct.instance(PARTITION_STATES);
- TopicPartition topicPartition = entry.getKey();
- partitionStateData.set(TOPIC_NAME, topicPartition.topic());
- partitionStateData.set(PARTITION_ID, topicPartition.partition());
- entry.getValue().setStruct(partitionStateData);
- partitionStatesData.add(partitionStateData);
- }
- struct.set(PARTITION_STATES, partitionStatesData.toArray());
- }
-
- List brokersData = new ArrayList<>(liveBrokers.size());
- for (Broker broker : liveBrokers) {
- Struct brokerData = struct.instance(LIVE_BROKERS);
- brokerData.set(BROKER_ID, broker.id);
-
- if (version == 0) {
- EndPoint endPoint = broker.endPoints.get(0);
- brokerData.set(HOST, endPoint.host);
- brokerData.set(PORT, endPoint.port);
- } else {
- List endPointsData = new ArrayList<>(broker.endPoints.size());
- for (EndPoint endPoint : broker.endPoints) {
- Struct endPointData = brokerData.instance(ENDPOINTS);
- endPointData.set(PORT, endPoint.port);
- endPointData.set(HOST, endPoint.host);
- endPointData.set(SECURITY_PROTOCOL_TYPE, endPoint.securityProtocol.id);
- if (version >= 3)
- endPointData.set(LISTENER_NAME, endPoint.listenerName.value());
- endPointsData.add(endPointData);
-
- }
- brokerData.set(ENDPOINTS, endPointsData.toArray());
- if (version >= 2) {
- brokerData.set(RACK, broker.rack);
- }
- }
+ if (version <= 5)
+ return new UpdateMetadataResponse(new UpdateMetadataResponseData().setErrorCode(Errors.forException(e).code()));
+ else
+ throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
+ version, this.getClass().getSimpleName(), ApiKeys.UPDATE_METADATA.latestVersion()));
+ }
- brokersData.add(brokerData);
+ public Iterable partitionStates() {
+ if (version() >= 5) {
+ return () -> new FlattenedIterator<>(data.topicStates().iterator(),
+ topicState -> topicState.partitionStates().iterator());
}
- struct.set(LIVE_BROKERS, brokersData.toArray());
+ return data.ungroupedPartitionStates();
+ }
- return struct;
+ public List liveBrokers() {
+ return data.liveBrokers();
}
@Override
- public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- short versionId = version();
- if (versionId <= 5)
- return new UpdateMetadataResponse(Errors.forException(e));
- else
- throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d",
- versionId, this.getClass().getSimpleName(), ApiKeys.UPDATE_METADATA.latestVersion()));
+ protected Struct toStruct() {
+ return data.toStruct(version());
}
- public Map partitionStates() {
- return partitionStates;
+ protected ByteBuffer toBytes() {
+ ByteBuffer bytes = ByteBuffer.allocate(size());
+ data.write(new ByteBufferAccessor(bytes), version());
+ bytes.flip();
+ return bytes;
}
- public Set liveBrokers() {
- return liveBrokers;
+ protected int size() {
+ return data.size(version());
}
public static UpdateMetadataRequest parse(ByteBuffer buffer, short version) {
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
index 478265b39586..c7d803f560eb 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
@@ -16,62 +16,41 @@
*/
package org.apache.kafka.common.requests;
+import org.apache.kafka.common.message.UpdateMetadataResponseData;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
import java.util.Map;
-import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE;
-
public class UpdateMetadataResponse extends AbstractResponse {
- private static final Schema UPDATE_METADATA_RESPONSE_V0 = new Schema(ERROR_CODE);
- private static final Schema UPDATE_METADATA_RESPONSE_V1 = UPDATE_METADATA_RESPONSE_V0;
- private static final Schema UPDATE_METADATA_RESPONSE_V2 = UPDATE_METADATA_RESPONSE_V1;
- private static final Schema UPDATE_METADATA_RESPONSE_V3 = UPDATE_METADATA_RESPONSE_V2;
- private static final Schema UPDATE_METADATA_RESPONSE_V4 = UPDATE_METADATA_RESPONSE_V3;
- private static final Schema UPDATE_METADATA_RESPONSE_V5 = UPDATE_METADATA_RESPONSE_V4;
-
- public static Schema[] schemaVersions() {
- return new Schema[]{UPDATE_METADATA_RESPONSE_V0, UPDATE_METADATA_RESPONSE_V1, UPDATE_METADATA_RESPONSE_V2,
- UPDATE_METADATA_RESPONSE_V3, UPDATE_METADATA_RESPONSE_V4, UPDATE_METADATA_RESPONSE_V5};
- }
- /**
- * Possible error code:
- *
- * STALE_CONTROLLER_EPOCH (11)
- * STALE_BROKER_EPOCH (77)
- */
- private final Errors error;
+ private final UpdateMetadataResponseData data;
- public UpdateMetadataResponse(Errors error) {
- this.error = error;
+ public UpdateMetadataResponse(UpdateMetadataResponseData data) {
+ this.data = data;
}
- public UpdateMetadataResponse(Struct struct) {
- error = Errors.forCode(struct.get(ERROR_CODE));
+ public UpdateMetadataResponse(Struct struct, short version) {
+ this(new UpdateMetadataResponseData(struct, version));
}
public Errors error() {
- return error;
+ return Errors.forCode(data.errorCode());
}
@Override
public Map errorCounts() {
- return errorCounts(error);
+ return errorCounts(error());
}
public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) {
- return new UpdateMetadataResponse(ApiKeys.UPDATE_METADATA.parseResponse(version, buffer));
+ return new UpdateMetadataResponse(ApiKeys.UPDATE_METADATA.parseResponse(version, buffer), version);
}
@Override
protected Struct toStruct(short version) {
- Struct struct = new Struct(ApiKeys.UPDATE_METADATA.responseSchema(version));
- struct.set(ERROR_CODE, error.code());
- return struct;
+ return data.toStruct(version);
}
-}
\ No newline at end of file
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index 3133f44a88f3..4bf56e80b3f6 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -23,6 +23,7 @@
import org.apache.kafka.common.errors.IllegalSaslStateException;
import org.apache.kafka.common.errors.SaslAuthenticationException;
import org.apache.kafka.common.errors.UnsupportedSaslMechanismException;
+import org.apache.kafka.common.message.RequestHeaderData;
import org.apache.kafka.common.message.SaslAuthenticateRequestData;
import org.apache.kafka.common.message.SaslHandshakeRequestData;
import org.apache.kafka.common.network.Authenticator;
@@ -324,7 +325,14 @@ public Long reauthenticationLatencyMs() {
private RequestHeader nextRequestHeader(ApiKeys apiKey, short version) {
String clientId = (String) configs.get(CommonClientConfigs.CLIENT_ID_CONFIG);
- currentRequestHeader = new RequestHeader(apiKey, version, clientId, correlationId++);
+ short requestApiKey = apiKey.id;
+ currentRequestHeader = new RequestHeader(
+ new RequestHeaderData().
+ setRequestApiKey(requestApiKey).
+ setRequestApiVersion(version).
+ setClientId(clientId).
+ setCorrelationId(correlationId++),
+ apiKey.headerVersion(version));
return currentRequestHeader;
}
diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
index 92a70b1f1945..91280ca65a34 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosRule.java
@@ -46,6 +46,7 @@ class KerberosRule {
private final String toPattern;
private final boolean repeat;
private final boolean toLowerCase;
+ private final boolean toUpperCase;
KerberosRule(String defaultRealm) {
this.defaultRealm = defaultRealm;
@@ -57,10 +58,11 @@ class KerberosRule {
toPattern = null;
repeat = false;
toLowerCase = false;
+ toUpperCase = false;
}
KerberosRule(String defaultRealm, int numOfComponents, String format, String match, String fromPattern,
- String toPattern, boolean repeat, boolean toLowerCase) {
+ String toPattern, boolean repeat, boolean toLowerCase, boolean toUpperCase) {
this.defaultRealm = defaultRealm;
isDefault = false;
this.numOfComponents = numOfComponents;
@@ -71,6 +73,7 @@ class KerberosRule {
this.toPattern = toPattern;
this.repeat = repeat;
this.toLowerCase = toLowerCase;
+ this.toUpperCase = toUpperCase;
}
@Override
@@ -102,6 +105,9 @@ public String toString() {
if (toLowerCase) {
buf.append("/L");
}
+ if (toUpperCase) {
+ buf.append("/U");
+ }
}
return buf.toString();
}
@@ -191,7 +197,10 @@ String apply(String[] params) throws IOException {
}
if (toLowerCase && result != null) {
result = result.toLowerCase(Locale.ENGLISH);
+ } else if (toUpperCase && result != null) {
+ result = result.toUpperCase(Locale.ENGLISH);
}
+
return result;
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java
index 69b4689009ba..96e01f17942b 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosShortNamer.java
@@ -33,7 +33,7 @@ public class KerberosShortNamer {
/**
* A pattern for parsing a auth_to_local rule.
*/
- private static final Pattern RULE_PARSER = Pattern.compile("((DEFAULT)|((RULE:\\[(\\d*):([^\\]]*)](\\(([^)]*)\\))?(s/([^/]*)/([^/]*)/(g)?)?/?(L)?)))");
+ private static final Pattern RULE_PARSER = Pattern.compile("((DEFAULT)|((RULE:\\[(\\d*):([^\\]]*)](\\(([^)]*)\\))?(s/([^/]*)/([^/]*)/(g)?)?/?(L|U)?)))");
/* Rules for the translation of the principal name into an operating system name */
private final List principalToLocalRules;
@@ -66,7 +66,8 @@ private static List parseRules(String defaultRealm, List r
matcher.group(10),
matcher.group(11),
"g".equals(matcher.group(12)),
- "L".equals(matcher.group(13))));
+ "L".equals(matcher.group(13)),
+ "U".equals(matcher.group(13))));
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java b/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java
index 6efe3112cdb4..ae9307506571 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/ByteUtils.java
@@ -129,7 +129,7 @@ public static void writeUnsignedIntLE(byte[] buffer, int offset, int value) {
}
/**
- * Read an integer stored in variable-length format using zig-zag decoding from
+ * Read an integer stored in variable-length format using unsigned decoding from
* Google Protocol Buffers.
*
* @param buffer The buffer to read from
@@ -137,7 +137,7 @@ public static void writeUnsignedIntLE(byte[] buffer, int offset, int value) {
*
* @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
*/
- public static int readVarint(ByteBuffer buffer) {
+ public static int readUnsignedVarint(ByteBuffer buffer) {
int value = 0;
int i = 0;
int b;
@@ -148,11 +148,11 @@ public static int readVarint(ByteBuffer buffer) {
throw illegalVarintException(value);
}
value |= b << i;
- return (value >>> 1) ^ -(value & 1);
+ return value;
}
/**
- * Read an integer stored in variable-length format using zig-zag decoding from
+ * Read an integer stored in variable-length format using unsigned decoding from
* Google Protocol Buffers.
*
* @param in The input to read from
@@ -161,7 +161,7 @@ public static int readVarint(ByteBuffer buffer) {
* @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
* @throws IOException if {@link DataInput} throws {@link IOException}
*/
- public static int readVarint(DataInput in) throws IOException {
+ public static int readUnsignedVarint(DataInput in) throws IOException {
int value = 0;
int i = 0;
int b;
@@ -172,6 +172,35 @@ public static int readVarint(DataInput in) throws IOException {
throw illegalVarintException(value);
}
value |= b << i;
+ return value;
+ }
+
+ /**
+ * Read an integer stored in variable-length format using zig-zag decoding from
+ * Google Protocol Buffers.
+ *
+ * @param buffer The buffer to read from
+ * @return The integer read
+ *
+ * @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
+ */
+ public static int readVarint(ByteBuffer buffer) {
+ int value = readUnsignedVarint(buffer);
+ return (value >>> 1) ^ -(value & 1);
+ }
+
+ /**
+ * Read an integer stored in variable-length format using zig-zag decoding from
+ * Google Protocol Buffers.
+ *
+ * @param in The input to read from
+ * @return The integer read
+ *
+ * @throws IllegalArgumentException if variable-length value does not terminate after 5 bytes have been read
+ * @throws IOException if {@link DataInput} throws {@link IOException}
+ */
+ public static int readVarint(DataInput in) throws IOException {
+ int value = readUnsignedVarint(in);
return (value >>> 1) ^ -(value & 1);
}
@@ -222,6 +251,40 @@ public static long readVarlong(ByteBuffer buffer) {
return (value >>> 1) ^ -(value & 1);
}
+ /**
+ * Write the given integer following the variable-length unsigned encoding from
+ * Google Protocol Buffers
+ * into the buffer.
+ *
+ * @param value The value to write
+ * @param buffer The output to write to
+ */
+ public static void writeUnsignedVarint(int value, ByteBuffer buffer) {
+ while ((value & 0xffffff80) != 0L) {
+ byte b = (byte) ((value & 0x7f) | 0x80);
+ buffer.put(b);
+ value >>>= 7;
+ }
+ buffer.put((byte) value);
+ }
+
+ /**
+ * Write the given integer following the variable-length unsigned encoding from
+ * Google Protocol Buffers
+ * into the buffer.
+ *
+ * @param value The value to write
+ * @param out The output to write to
+ */
+ public static void writeUnsignedVarint(int value, DataOutput out) throws IOException {
+ while ((value & 0xffffff80) != 0L) {
+ byte b = (byte) ((value & 0x7f) | 0x80);
+ out.writeByte(b);
+ value >>>= 7;
+ }
+ out.writeByte((byte) value);
+ }
+
/**
* Write the given integer following the variable-length zig-zag encoding from
* Google Protocol Buffers
@@ -231,12 +294,7 @@ public static long readVarlong(ByteBuffer buffer) {
* @param out The output to write to
*/
public static void writeVarint(int value, DataOutput out) throws IOException {
- int v = (value << 1) ^ (value >> 31);
- while ((v & 0xffffff80) != 0L) {
- out.writeByte((v & 0x7f) | 0x80);
- v >>>= 7;
- }
- out.writeByte((byte) v);
+ writeUnsignedVarint((value << 1) ^ (value >> 31), out);
}
/**
@@ -248,13 +306,7 @@ public static void writeVarint(int value, DataOutput out) throws IOException {
* @param buffer The output to write to
*/
public static void writeVarint(int value, ByteBuffer buffer) {
- int v = (value << 1) ^ (value >> 31);
- while ((v & 0xffffff80) != 0L) {
- byte b = (byte) ((v & 0x7f) | 0x80);
- buffer.put(b);
- v >>>= 7;
- }
- buffer.put((byte) v);
+ writeUnsignedVarint((value << 1) ^ (value >> 31), buffer);
}
/**
@@ -293,20 +345,28 @@ public static void writeVarlong(long value, ByteBuffer buffer) {
}
/**
- * Number of bytes needed to encode an integer in variable-length format.
+ * Number of bytes needed to encode an integer in unsigned variable-length format.
*
* @param value The signed value
*/
- public static int sizeOfVarint(int value) {
- int v = (value << 1) ^ (value >> 31);
+ public static int sizeOfUnsignedVarint(int value) {
int bytes = 1;
- while ((v & 0xffffff80) != 0L) {
+ while ((value & 0xffffff80) != 0L) {
bytes += 1;
- v >>>= 7;
+ value >>>= 7;
}
return bytes;
}
+ /**
+ * Number of bytes needed to encode an integer in variable-length format.
+ *
+ * @param value The signed value
+ */
+ public static int sizeOfVarint(int value) {
+ return sizeOfUnsignedVarint((value << 1) ^ (value >> 31));
+ }
+
/**
* Number of bytes needed to encode a long in variable-length format.
*
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java b/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java
new file mode 100644
index 000000000000..48bf3b7199e1
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import java.util.Iterator;
+import java.util.function.Function;
+
+/**
+ * Provides a flattened iterator over the inner elements of an outer iterator.
+ */
+public final class FlattenedIterator extends AbstractIterator {
+ private final Iterator outerIterator;
+ private final Function> innerIteratorFunction;
+ private Iterator innerIterator;
+
+ public FlattenedIterator(Iterator outerIterator, Function> innerIteratorFunction) {
+ this.outerIterator = outerIterator;
+ this.innerIteratorFunction = innerIteratorFunction;
+ }
+
+ @Override
+ public I makeNext() {
+ while (innerIterator == null || !innerIterator.hasNext()) {
+ if (outerIterator.hasNext())
+ innerIterator = innerIteratorFunction.apply(outerIterator.next());
+ else
+ return allDone();
+ }
+ return innerIterator.next();
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/MappedIterator.java b/clients/src/main/java/org/apache/kafka/common/utils/MappedIterator.java
new file mode 100644
index 000000000000..f6eb270c56a2
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/utils/MappedIterator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.utils;
+
+import java.util.Iterator;
+import java.util.function.Function;
+
+/**
+ * An iterator that maps another iterator's elements from type `F` to type `T`.
+ */
+public final class MappedIterator implements Iterator {
+ private final Iterator underlyingIterator;
+ private final Function mapper;
+
+ public MappedIterator(Iterator underlyingIterator, Function mapper) {
+ this.underlyingIterator = underlyingIterator;
+ this.mapper = mapper;
+ }
+
+ @Override
+ public final boolean hasNext() {
+ return underlyingIterator.hasNext();
+ }
+
+ @Override
+ public final T next() {
+ return mapper.apply(underlyingIterator.next());
+ }
+
+}
diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
index 49c7290106af..f68b9381ce1d 100644
--- a/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
+++ b/clients/src/main/java/org/apache/kafka/server/authorizer/AuthorizableRequestContext.java
@@ -32,7 +32,7 @@ public interface AuthorizableRequestContext {
/**
* Returns name of listener on which request was received.
*/
- String listener();
+ String listenerName();
/**
* Returns the security protocol for the listener on which request was received.
diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json
index d2285668f4cb..e396ed209643 100644
--- a/clients/src/main/resources/common/message/CreateTopicsRequest.json
+++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json
@@ -19,7 +19,8 @@
"name": "CreateTopicsRequest",
// Version 1 adds validateOnly.
// Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
- "validVersions": "0-4",
+ // Version 5 returns topic configs in the response (KIP-525).
+ "validVersions": "0-5",
"fields": [
{ "name": "Topics", "type": "[]CreatableTopic", "versions": "0+",
"about": "The topics to create.", "fields": [
diff --git a/clients/src/main/resources/common/message/CreateTopicsResponse.json b/clients/src/main/resources/common/message/CreateTopicsResponse.json
index d56e642061e4..f646258bff5a 100644
--- a/clients/src/main/resources/common/message/CreateTopicsResponse.json
+++ b/clients/src/main/resources/common/message/CreateTopicsResponse.json
@@ -20,8 +20,9 @@
// Version 1 adds a per-topic error message string.
// Version 2 adds the throttle time.
// Starting in version 3, on quota violation, brokers send out responses before throttling.
- // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464)
- "validVersions": "0-4",
+ // Version 4 makes partitions/replicationFactor optional even when assignments are not present (KIP-464).
+ // Version 5 returns topic configs in the response (KIP-525)
+ "validVersions": "0-5",
"fields": [
{ "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true,
"about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
@@ -32,7 +33,26 @@
{ "name": "ErrorCode", "type": "int16", "versions": "0+",
"about": "The error code, or 0 if there was no error." },
{ "name": "ErrorMessage", "type": "string", "versions": "1+", "nullableVersions": "0+", "ignorable": true,
- "about": "The error message, or null if there was no error." }
+ "about": "The error message, or null if there was no error." },
+ { "name": "TopicConfigErrorCode", "type": "int16", "versions": "5+", "default": "0",
+ "about": "Optional topic config error returned if configs are not returned in the response." },
+ { "name": "NumPartitions", "type": "int32", "versions": "5+", "default": "-1",
+ "about": "Number of partitions of the topic." },
+ { "name": "ReplicationFactor", "type": "int16", "versions": "5+", "default": "-1",
+ "about": "Replicator factor of the topic." },
+ { "name": "Configs", "type": "[]CreatableTopicConfigs", "versions": "5+", "nullableVersions": "5+",
+ "about": "Configuration of the topic.", "fields": [
+ { "name": "Name", "type": "string", "versions": "5+",
+ "about": "The configuration name." },
+ { "name": "Value", "type": "string", "versions": "5+", "nullableVersions": "0+",
+ "about": "The configuration value." },
+ { "name": "ReadOnly", "type": "bool", "versions": "5+",
+ "about": "True if the configuration is read-only." },
+ { "name": "ConfigSource", "type": "int8", "versions": "5+", "default": "-1", "ignorable": true,
+ "about": "The configuration source." },
+ { "name": "IsSensitive", "type": "bool", "versions": "5+",
+ "about": "True if this configuration is sensitive." }
+ ]}
]}
]
}
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
index c43d2f4b1633..12f159a15740 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
@@ -30,16 +30,16 @@
"about": "The current controller epoch." },
{ "name": "BrokerEpoch", "type": "int64", "versions": "2+", "ignorable": true, "default": "-1",
"about": "The current broker epoch." },
- { "name": "PartitionStatesV0", "type": "[]LeaderAndIsrRequestPartition", "versions": "0-1",
+ { "name": "UngroupedPartitionStates", "type": "[]LeaderAndIsrPartitionState", "versions": "0-1",
"about": "The state of each partition, in a v0 or v1 message." },
// In v0 or v1 requests, each partition is listed alongside its topic name.
// In v2+ requests, partitions are organized by topic, so that each topic name
// only needs to be listed once.
- { "name": "TopicStates", "type": "[]LeaderAndIsrRequestTopicState", "versions": "2+",
+ { "name": "TopicStates", "type": "[]LeaderAndIsrTopicState", "versions": "2+",
"about": "Each topic.", "fields": [
- { "name": "Name", "type": "string", "versions": "2+", "entityType": "topicName",
+ { "name": "TopicName", "type": "string", "versions": "2+", "entityType": "topicName",
"about": "The topic name." },
- { "name": "PartitionStatesV0", "type": "[]LeaderAndIsrRequestPartition", "versions": "2+",
+ { "name": "PartitionStates", "type": "[]LeaderAndIsrPartitionState", "versions": "2+",
"about": "The state of each partition" }
]},
{ "name": "LiveLeaders", "type": "[]LeaderAndIsrLiveLeader", "versions": "0+",
@@ -53,18 +53,18 @@
]}
],
"commonStructs": [
- { "name": "LeaderAndIsrRequestPartition", "versions": "0+", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0-1", "entityType": "topicName",
+ { "name": "LeaderAndIsrPartitionState", "versions": "0+", "fields": [
+ { "name": "TopicName", "type": "string", "versions": "0-1", "entityType": "topicName", "ignorable": true,
"about": "The topic name. This is only present in v0 or v1." },
{ "name": "PartitionIndex", "type": "int32", "versions": "0+",
"about": "The partition index." },
{ "name": "ControllerEpoch", "type": "int32", "versions": "0+",
"about": "The controller epoch." },
- { "name": "LeaderKey", "type": "int32", "versions": "0+", "entityType": "brokerId",
+ { "name": "Leader", "type": "int32", "versions": "0+", "entityType": "brokerId",
"about": "The broker ID of the leader." },
{ "name": "LeaderEpoch", "type": "int32", "versions": "0+",
"about": "The leader epoch." },
- { "name": "IsrReplicas", "type": "[]int32", "versions": "0+",
+ { "name": "Isr", "type": "[]int32", "versions": "0+",
"about": "The in-sync replica IDs." },
{ "name": "ZkVersion", "type": "int32", "versions": "0+",
"about": "The ZooKeeper version." },
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
index 06bb088e179b..f30055b97aac 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
@@ -26,7 +26,7 @@
"fields": [
{ "name": "ErrorCode", "type": "int16", "versions": "0+",
"about": "The error code, or 0 if there was no error." },
- { "name": "Partitions", "type": "[]LeaderAndIsrResponsePartition", "versions": "0+",
+ { "name": "PartitionErrors", "type": "[]LeaderAndIsrPartitionError", "versions": "0+",
"about": "Each partition.", "fields": [
{ "name": "TopicName", "type": "string", "versions": "0+", "entityType": "topicName",
"about": "The topic name." },
diff --git a/clients/src/main/resources/common/message/OffsetDeleteRequest.json b/clients/src/main/resources/common/message/OffsetDeleteRequest.json
new file mode 100644
index 000000000000..563594997097
--- /dev/null
+++ b/clients/src/main/resources/common/message/OffsetDeleteRequest.json
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+ "apiKey": 47,
+ "type": "request",
+ "name": "OffsetDeleteRequest",
+ "validVersions": "0",
+ "fields": [
+ { "name": "GroupId", "type": "string", "versions": "0+",
+ "about": "The unique group identifier." },
+ { "name": "Topics", "type": "[]OffsetDeleteRequestTopic", "versions": "0+",
+ "about": "The topics to delete offsets for", "fields": [
+ { "name": "Name", "type": "string", "versions": "0+", "mapKey": true,
+ "about": "The topic name." },
+ { "name": "Partitions", "type": "[]OffsetDeleteRequestPartition", "versions": "0+",
+ "about": "Each partition to delete offsets for.", "fields": [
+ { "name": "PartitionIndex", "type": "int32", "versions": "0+",
+ "about": "The partition index." }
+ ]
+ }
+ ]
+ }
+ ]
+}
diff --git a/clients/src/main/resources/common/message/OffsetDeleteResponse.json b/clients/src/main/resources/common/message/OffsetDeleteResponse.json
new file mode 100644
index 000000000000..f2be321f5d8f
--- /dev/null
+++ b/clients/src/main/resources/common/message/OffsetDeleteResponse.json
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+ "apiKey": 47,
+ "type": "response",
+ "name": "OffsetDeleteResponse",
+ "validVersions": "0",
+ "fields": [
+ { "name": "ErrorCode", "type": "int16", "versions": "0+",
+ "about": "The top-level error code, or 0 if there was no error." },
+ { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "ignorable": true,
+ "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
+ { "name": "Topics", "type": "[]OffsetDeleteResponseTopic", "versions": "0+",
+ "about": "The responses for each topic.", "fields": [
+ { "name": "Name", "type": "string", "versions": "0+", "mapKey": true,
+ "about": "The topic name." },
+ { "name": "Partitions", "type": "[]OffsetDeleteResponsePartition", "versions": "0+",
+ "about": "The responses for each partition in the topic.", "fields": [
+ { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true,
+ "about": "The partition index." },
+ { "name": "ErrorCode", "type": "int16", "versions": "0+",
+ "about": "The error code, or 0 if there was no error." }
+ ]
+ }
+ ]
+ }
+ ]
+}
diff --git a/clients/src/main/resources/common/message/RequestHeader.json b/clients/src/main/resources/common/message/RequestHeader.json
index d24dcf0e9875..a00658cc866d 100644
--- a/clients/src/main/resources/common/message/RequestHeader.json
+++ b/clients/src/main/resources/common/message/RequestHeader.json
@@ -16,7 +16,10 @@
{
"type": "header",
"name": "RequestHeader",
- "validVersions": "0",
+ // Version 0 of the RequestHeader is only used by v0 of ControlledShutdownRequest.
+ //
+ // Version 1 is the first version with ClientId.
+ "validVersions": "0-1",
"fields": [
{ "name": "RequestApiKey", "type": "int16", "versions": "0+",
"about": "The API key of this request." },
@@ -24,7 +27,7 @@
"about": "The API version of this request." },
{ "name": "CorrelationId", "type": "int32", "versions": "0+",
"about": "The correlation ID of this request." },
- { "name": "ClientId", "type": "string", "versions": "0+",
+ { "name": "ClientId", "type": "string", "versions": "1+", "nullableVersions": "1+", "ignorable": true,
"about": "The client ID string." }
]
}
diff --git a/clients/src/main/resources/common/message/ResponseHeader.json b/clients/src/main/resources/common/message/ResponseHeader.json
index d44825977da7..bc47b3f05f07 100644
--- a/clients/src/main/resources/common/message/ResponseHeader.json
+++ b/clients/src/main/resources/common/message/ResponseHeader.json
@@ -16,7 +16,8 @@
{
"type": "header",
"name": "ResponseHeader",
- "validVersions": "0",
+ // Version 0 and version 1 are the same.
+ "validVersions": "0-1",
"fields": [
{ "name": "CorrelationId", "type": "int32", "versions": "0+",
"about": "The correlation ID of this response." }
diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json
index 12c3d7e8feb6..248faa58a1f8 100644
--- a/clients/src/main/resources/common/message/StopReplicaRequest.json
+++ b/clients/src/main/resources/common/message/StopReplicaRequest.json
@@ -29,14 +29,14 @@
"about": "The broker epoch." },
{ "name": "DeletePartitions", "type": "bool", "versions": "0+",
"about": "Whether these partitions should be deleted." },
- { "name": "PartitionsV0", "type": "[]StopReplicaRequestPartitionV0", "versions": "0",
+ { "name": "UngroupedPartitions", "type": "[]StopReplicaPartitionV0", "versions": "0",
"about": "The partitions to stop.", "fields": [
{ "name": "TopicName", "type": "string", "versions": "0", "entityType": "topicName",
"about": "The topic name." },
{ "name": "PartitionIndex", "type": "int32", "versions": "0",
"about": "The partition index." }
]},
- { "name": "Topics", "type": "[]StopReplicaRequestTopic", "versions": "1+",
+ { "name": "Topics", "type": "[]StopReplicaTopic", "versions": "1+",
"about": "The topics to stop.", "fields": [
{ "name": "Name", "type": "string", "versions": "1+", "entityType": "topicName",
"about": "The topic name." },
diff --git a/clients/src/main/resources/common/message/StopReplicaResponse.json b/clients/src/main/resources/common/message/StopReplicaResponse.json
index 962cacbedd8b..759bffecf07c 100644
--- a/clients/src/main/resources/common/message/StopReplicaResponse.json
+++ b/clients/src/main/resources/common/message/StopReplicaResponse.json
@@ -22,7 +22,7 @@
"fields": [
{ "name": "ErrorCode", "type": "int16", "versions": "0+",
"about": "The top-level error code, or 0 if there was no top-level error." },
- { "name": "Partitions", "type": "[]StopReplicaResponsePartition", "versions": "0+",
+ { "name": "PartitionErrors", "type": "[]StopReplicaPartitionError", "versions": "0+",
"about": "The responses for each partition.", "fields": [
{ "name": "TopicName", "type": "string", "versions": "0+", "entityType": "topicName",
"about": "The topic name." },
diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
index d0873d1fd097..b1ac987b46c6 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
@@ -34,16 +34,16 @@
"about": "The controller epoch." },
{ "name": "BrokerEpoch", "type": "int64", "versions": "5+", "ignorable": true, "default": "-1",
"about": "The broker epoch." },
- { "name": "LegacyPartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "0-4",
+ { "name": "UngroupedPartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "0-4",
"about": "In older versions of this RPC, each partition that we would like to update." },
- { "name": "TopicStates", "type": "[]UpdateMetadataRequestTopicState", "versions": "5+",
+ { "name": "TopicStates", "type": "[]UpdateMetadataTopicState", "versions": "5+",
"about": "In newer versions of this RPC, each topic that we would like to update.", "fields": [
{ "name": "TopicName", "type": "string", "versions": "5+", "entityType": "topicName",
"about": "The topic name." },
{ "name": "PartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "5+",
"about": "The partition that we would like to update." }
]},
- { "name": "Brokers", "type": "[]UpdateMetadataRequestBroker", "versions": "0+", "fields": [
+ { "name": "LiveBrokers", "type": "[]UpdateMetadataBroker", "versions": "0+", "fields": [
{ "name": "Id", "type": "int32", "versions": "0+", "entityType": "brokerId",
"about": "The broker id." },
// Version 0 of the protocol only allowed specifying a single host and
@@ -52,13 +52,13 @@
"about": "The broker hostname." },
{ "name": "V0Port", "type": "int32", "versions": "0", "ignorable": true,
"about": "The broker port." },
- { "name": "Endpoints", "type": "[]UpdateMetadataRequestEndpoint", "versions": "1+",
+ { "name": "Endpoints", "type": "[]UpdateMetadataEndpoint", "versions": "1+", "ignorable": true,
"about": "The broker endpoints.", "fields": [
{ "name": "Port", "type": "int32", "versions": "1+",
"about": "The port of this endpoint" },
{ "name": "Host", "type": "string", "versions": "1+",
"about": "The hostname of this endpoint" },
- { "name": "Listener", "type": "string", "versions": "3+",
+ { "name": "Listener", "type": "string", "versions": "3+", "ignorable": true,
"about": "The listener name." },
{ "name": "SecurityProtocol", "type": "int16", "versions": "1+",
"about": "The security protocol type." }
@@ -69,7 +69,7 @@
],
"commonStructs": [
{ "name": "UpdateMetadataPartitionState", "versions": "0+", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0-4", "entityType": "topicName",
+ { "name": "TopicName", "type": "string", "versions": "0-4", "entityType": "topicName", "ignorable": true,
"about": "In older versions of this RPC, the topic name." },
{ "name": "PartitionIndex", "type": "int32", "versions": "0+",
"about": "The partition index." },
@@ -85,7 +85,7 @@
"about": "The Zookeeper version." },
{ "name": "Replicas", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
"about": "All the replicas of this partition." },
- { "name": "OfflineReplicas", "type": "[]int32", "versions": "4+", "entityType": "brokerId",
+ { "name": "OfflineReplicas", "type": "[]int32", "versions": "4+", "entityType": "brokerId", "ignorable": true,
"about": "The replicas of this partition which are offline." }
]}
]
diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
index b5bba716296b..ab11b95520f9 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
@@ -45,6 +45,7 @@
import java.util.Collections;
import java.util.List;
+import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -155,16 +156,23 @@ public void testUnsupportedVersionDuringInternalMetadataRequest() {
private void checkSimpleRequestResponse(NetworkClient networkClient) {
awaitReady(networkClient, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0
- ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000,
- Collections.emptyMap());
+ ProduceRequest.Builder builder = new ProduceRequest.Builder(
+ PRODUCE.latestVersion(),
+ PRODUCE.latestVersion(),
+ (short) 1,
+ 1000,
+ Collections.emptyMap(),
+ null);
TestCallbackHandler handler = new TestCallbackHandler();
ClientRequest request = networkClient.newClientRequest(
node.idString(), builder, time.milliseconds(), true, defaultRequestTimeoutMs, handler);
networkClient.send(request, time.milliseconds());
networkClient.poll(1, time.milliseconds());
assertEquals(1, networkClient.inFlightRequestCount());
- ResponseHeader respHeader = new ResponseHeader(request.correlationId());
- Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion()));
+ ResponseHeader respHeader =
+ new ResponseHeader(request.correlationId(),
+ request.apiKey().headerVersion(PRODUCE.latestVersion()));
+ Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
resp.set("responses", new Object[0]);
Struct responseHeaderStruct = respHeader.toStruct();
int size = responseHeaderStruct.sizeOf() + resp.sizeOf();
@@ -183,7 +191,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) {
private void setExpectedApiVersionsResponse(ApiVersionsResponse response) {
short apiVersionsResponseVersion = response.apiVersion(ApiKeys.API_VERSIONS.id).maxVersion;
- ByteBuffer buffer = response.serialize(apiVersionsResponseVersion, new ResponseHeader(0));
+ ByteBuffer buffer = response.serialize(ApiKeys.API_VERSIONS, apiVersionsResponseVersion, 0);
selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
}
@@ -235,15 +243,22 @@ private void testRequestTimeout(ClientRequest request) {
public void testConnectionThrottling() {
// Instrument the test to return a response with a 100ms throttle delay.
awaitReady(client, node);
- ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000,
- Collections.emptyMap());
+ ProduceRequest.Builder builder = new ProduceRequest.Builder(
+ PRODUCE.latestVersion(),
+ PRODUCE.latestVersion(),
+ (short) 1,
+ 1000,
+ Collections.emptyMap(),
+ null);
TestCallbackHandler handler = new TestCallbackHandler();
ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true,
defaultRequestTimeoutMs, handler);
client.send(request, time.milliseconds());
client.poll(1, time.milliseconds());
- ResponseHeader respHeader = new ResponseHeader(request.correlationId());
- Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion()));
+ ResponseHeader respHeader =
+ new ResponseHeader(request.correlationId(),
+ request.apiKey().headerVersion(PRODUCE.latestVersion()));
+ Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
resp.set("responses", new Object[0]);
resp.set(CommonFields.THROTTLE_TIME_MS, 100);
Struct responseHeaderStruct = respHeader.toStruct();
@@ -288,7 +303,7 @@ private ApiVersionsResponse createExpectedApiVersionsResponse(ApiKeys key, short
public void testThrottlingNotEnabledForConnectionToOlderBroker() {
// Instrument the test so that the max protocol version for PRODUCE returned from the node is 5 and thus
// client-side throttling is not enabled. Also, return a response with a 100ms throttle delay.
- setExpectedApiVersionsResponse(createExpectedApiVersionsResponse(ApiKeys.PRODUCE, (short) 5));
+ setExpectedApiVersionsResponse(createExpectedApiVersionsResponse(PRODUCE, (short) 5));
while (!client.ready(node, time.milliseconds()))
client.poll(1, time.milliseconds());
selector.clear();
@@ -316,8 +331,7 @@ private int sendEmptyProduceRequest() {
}
- private void sendResponse(int correlationId, Struct response) {
- ResponseHeader respHeader = new ResponseHeader(correlationId);
+ private void sendResponse(ResponseHeader respHeader, Struct response) {
Struct responseHeaderStruct = respHeader.toStruct();
int size = responseHeaderStruct.sizeOf() + response.sizeOf();
ByteBuffer buffer = ByteBuffer.allocate(size);
@@ -328,10 +342,12 @@ private void sendResponse(int correlationId, Struct response) {
}
private void sendThrottledProduceResponse(int correlationId, int throttleMs) {
- Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion()));
+ Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion()));
resp.set("responses", new Object[0]);
resp.set(CommonFields.THROTTLE_TIME_MS, throttleMs);
- sendResponse(correlationId, resp);
+ sendResponse(new ResponseHeader(correlationId,
+ PRODUCE.headerVersion(PRODUCE.latestVersion())),
+ resp);
}
@Test
diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
index 5d4d84b2d47c..b6fece991d24 100644
--- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.kafka.clients.admin;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.kafka.clients.ClientDnsLookup;
import org.apache.kafka.clients.ClientUtils;
import org.apache.kafka.clients.MockClient;
@@ -41,6 +43,7 @@
import org.apache.kafka.common.errors.AuthenticationException;
import org.apache.kafka.common.errors.ClusterAuthorizationException;
import org.apache.kafka.common.errors.GroupAuthorizationException;
+import org.apache.kafka.common.errors.GroupSubscribedToTopicException;
import org.apache.kafka.common.errors.InvalidRequestException;
import org.apache.kafka.common.errors.InvalidTopicException;
import org.apache.kafka.common.errors.LeaderNotAvailableException;
@@ -72,6 +75,11 @@
import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
import org.apache.kafka.common.message.ListGroupsResponseData;
import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartition;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic;
+import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse;
import org.apache.kafka.common.requests.ApiError;
@@ -98,6 +106,7 @@
import org.apache.kafka.common.requests.ListPartitionReassignmentsResponse;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.MetadataResponse;
+import org.apache.kafka.common.requests.OffsetDeleteResponse;
import org.apache.kafka.common.requests.OffsetFetchResponse;
import org.apache.kafka.common.resource.PatternType;
import org.apache.kafka.common.resource.ResourcePattern;
@@ -242,6 +251,30 @@ public void testCloseAdminClient() {
}
}
+ private static OffsetDeleteResponse prepareOffsetDeleteResponse(Errors error) {
+ return new OffsetDeleteResponse(
+ new OffsetDeleteResponseData()
+ .setErrorCode(error.code())
+ .setTopics(new OffsetDeleteResponseTopicCollection())
+ );
+ }
+
+ private static OffsetDeleteResponse prepareOffsetDeleteResponse(String topic, int partition, Errors error) {
+ return new OffsetDeleteResponse(
+ new OffsetDeleteResponseData()
+ .setErrorCode(Errors.NONE.code())
+ .setTopics(new OffsetDeleteResponseTopicCollection(Stream.of(
+ new OffsetDeleteResponseTopic()
+ .setName(topic)
+ .setPartitions(new OffsetDeleteResponsePartitionCollection(Collections.singletonList(
+ new OffsetDeleteResponsePartition()
+ .setPartitionIndex(partition)
+ .setErrorCode(error.code())
+ ).iterator()))
+ ).collect(Collectors.toList()).iterator()))
+ );
+ }
+
private static CreateTopicsResponse prepareCreateTopicsResponse(String topicName, Errors error) {
CreateTopicsResponseData data = new CreateTopicsResponseData();
data.topics().add(new CreatableTopicResult().
@@ -1522,6 +1555,226 @@ public void testDeleteConsumerGroups() throws Exception {
}
}
+ @Test
+ public void testDeleteConsumerGroupOffsets() throws Exception {
+ // Happy path
+
+ final Map nodes = new HashMap<>();
+ nodes.put(0, new Node(0, "localhost", 8121));
+
+ final Cluster cluster =
+ new Cluster(
+ "mockClusterId",
+ nodes.values(),
+ Collections.emptyList(),
+ Collections.emptySet(),
+ Collections.emptySet(), nodes.get(0));
+
+ final String groupId = "group-0";
+ final TopicPartition tp1 = new TopicPartition("foo", 0);
+ final TopicPartition tp2 = new TopicPartition("bar", 0);
+ final TopicPartition tp3 = new TopicPartition("foobar", 0);
+
+ try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
+ env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+ env.kafkaClient().prepareResponse(
+ prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+
+ env.kafkaClient().prepareResponse(new OffsetDeleteResponse(
+ new OffsetDeleteResponseData()
+ .setTopics(new OffsetDeleteResponseTopicCollection(Stream.of(
+ new OffsetDeleteResponseTopic()
+ .setName("foo")
+ .setPartitions(new OffsetDeleteResponsePartitionCollection(Collections.singletonList(
+ new OffsetDeleteResponsePartition()
+ .setPartitionIndex(0)
+ .setErrorCode(Errors.NONE.code())
+ ).iterator())),
+ new OffsetDeleteResponseTopic()
+ .setName("bar")
+ .setPartitions(new OffsetDeleteResponsePartitionCollection(Collections.singletonList(
+ new OffsetDeleteResponsePartition()
+ .setPartitionIndex(0)
+ .setErrorCode(Errors.GROUP_SUBSCRIBED_TO_TOPIC.code())
+ ).iterator()))
+ ).collect(Collectors.toList()).iterator()))
+ )
+ );
+
+ final DeleteConsumerGroupOffsetsResult errorResult = env.adminClient().deleteConsumerGroupOffsets(
+ groupId, Stream.of(tp1, tp2).collect(Collectors.toSet()));
+
+ assertNull(errorResult.all().get());
+ assertNull(errorResult.partitionResult(tp1).get());
+ TestUtils.assertFutureError(errorResult.partitionResult(tp2), GroupSubscribedToTopicException.class);
+ TestUtils.assertFutureError(errorResult.partitionResult(tp3), IllegalArgumentException.class);
+ }
+ }
+
+ @Test
+ public void testDeleteConsumerGroupOffsetsRetriableErrors() throws Exception {
+ // Retriable errors should be retried
+
+ final Map nodes = new HashMap<>();
+ nodes.put(0, new Node(0, "localhost", 8121));
+
+ final Cluster cluster =
+ new Cluster(
+ "mockClusterId",
+ nodes.values(),
+ Collections.emptyList(),
+ Collections.emptySet(),
+ Collections.emptySet(), nodes.get(0));
+
+ final String groupId = "group-0";
+ final TopicPartition tp1 = new TopicPartition("foo", 0);
+
+ try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
+ env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
+
+ env.kafkaClient().prepareResponse(
+ FindCoordinatorResponse.prepareResponse(Errors.NONE, env.cluster().controller()));
+
+ env.kafkaClient().prepareResponse(
+ prepareOffsetDeleteResponse(Errors.COORDINATOR_NOT_AVAILABLE));
+
+ env.kafkaClient().prepareResponse(
+ prepareOffsetDeleteResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS));
+
+ /*
+ * We need to return two responses here, one for NOT_COORDINATOR call when calling delete a consumer group
+ * api using coordinator that has moved. This will retry whole operation. So we need to again respond with a
+ * FindCoordinatorResponse.
+ */
+ env.kafkaClient().prepareResponse(
+ prepareOffsetDeleteResponse(Errors.NOT_COORDINATOR));
+
+ env.kafkaClient().prepareResponse(
+ prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
+
+ env.kafkaClient().prepareResponse(
+ prepareOffsetDeleteResponse("foo", 0, Errors.NONE));
+
+ final DeleteConsumerGroupOffsetsResult errorResult1 = env.adminClient()
+ .deleteConsumerGroupOffsets(groupId, Stream.of(tp1).collect(Collectors.toSet()));
+
+ assertNull(errorResult1.all().get());
+ assertNull(errorResult1.partitionResult(tp1).get());
+ }
+ }
+
+ @Test
+ public void testDeleteConsumerGroupOffsetsNonRetriableErrors() throws Exception {
+ // Non-retriable errors throw an exception
+
+ final Map nodes = new HashMap<>();
+ nodes.put(0, new Node(0, "localhost", 8121));
+
+ final Cluster cluster =
+ new Cluster(
+ "mockClusterId",
+ nodes.values(),
+ Collections.