- Starting in Kafka Streams 2.6.x, a new processing mode is available, named EOS version 2. This can be configured
- by setting "processing.guarantee" to "exactly_once_v2" for
- application versions 3.0+, or setting it to "exactly_once_beta" for versions between 2.6 and 2.8.
- To use this new feature, your brokers must be on version 2.5.x or newer.
- If you want to upgrade your EOS application from an older version and enable this feature in version 3.0+,
- you first need to upgrade your application to version 3.0.x, staying on "exactly_once",
- and then do second round of rolling bounces to switch to "exactly_once_v2". If you
+ Starting in Kafka Streams 4.0.0, the only processing modes available are at_least_once and
+ "exactly_once_v2". To use the latter, your brokers must be on version 2.5.x or newer.
+ Previous versions of Kafka Streams allowed "exactly_once" and "exactly_once_beta"
+ settings, which were deprecated in 3.0.0 and removed in 4.0.0. If you want to upgrade your application that uses
+ "exactly_once" in a version <3.0 and enable this feature in version 3.0+, you first need to
+ upgrade your application to version 3.0.x, staying on "exactly_once",
+ and then do second round of rolling bounces to switch to exactly_once_v2. If you
are upgrading an EOS application from an older (pre-2.6) version to a version between 2.6 and 2.8, follow these
same steps but with the config "exactly_once_beta" instead. No special steps are required
to upgrade an application using "exactly_once_beta" from version 2.6+ to 3.0 or higher: you can
@@ -105,6 +105,18 @@
We improved the semantics of
diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
index 325cbec90d9d..c5b151465f92 100644
--- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
+++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java
@@ -129,8 +129,7 @@
*
*
* If {@link #PROCESSING_GUARANTEE_CONFIG "processing.guarantee"} is set to {@link #EXACTLY_ONCE_V2 "exactly_once_v2"},
- * {@link #EXACTLY_ONCE "exactly_once"} (deprecated), or {@link #EXACTLY_ONCE_BETA "exactly_once_beta"} (deprecated), Kafka Streams does not
- * allow users to overwrite the following properties (Streams setting shown in parentheses):
+ * Kafka Streams does not allow users to overwrite the following properties (Streams setting shown in parentheses):
*
* - {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG "isolation.level"} (read_committed) - Consumers will always read committed data only
* - {@link ProducerConfig#ENABLE_IDEMPOTENCE_CONFIG "enable.idempotence"} (true) - Producer will always have idempotency enabled
@@ -357,32 +356,6 @@ public class StreamsConfig extends AbstractConfig {
@SuppressWarnings("WeakerAccess")
public static final String AT_LEAST_ONCE = "at_least_once";
- /**
- * Config value for parameter {@link #PROCESSING_GUARANTEE_CONFIG "processing.guarantee"} for exactly-once processing guarantees.
- *
- * Enabling exactly-once processing semantics requires broker version 0.11.0 or higher.
- * If you enable this feature Kafka Streams will use more resources (like broker connections)
- * compared to {@link #AT_LEAST_ONCE "at_least_once"} and {@link #EXACTLY_ONCE_V2 "exactly_once_v2"}.
- *
- * @deprecated Since 3.0.0, will be removed in 4.0. Use {@link #EXACTLY_ONCE_V2 "exactly_once_v2"} instead.
- */
- @SuppressWarnings("WeakerAccess")
- @Deprecated
- public static final String EXACTLY_ONCE = "exactly_once";
-
- /**
- * Config value for parameter {@link #PROCESSING_GUARANTEE_CONFIG "processing.guarantee"} for exactly-once processing guarantees.
- *
- * Enabling exactly-once (beta) requires broker version 2.5 or higher.
- * If you enable this feature Kafka Streams will use fewer resources (like broker connections)
- * compared to the {@link #EXACTLY_ONCE} (deprecated) case.
- *
- * @deprecated Since 3.0.0, will be removed in 4.0. Use {@link #EXACTLY_ONCE_V2 "exactly_once_v2"} instead.
- */
- @SuppressWarnings("WeakerAccess")
- @Deprecated
- public static final String EXACTLY_ONCE_BETA = "exactly_once_beta";
-
/**
* Config value for parameter {@link #PROCESSING_GUARANTEE_CONFIG "processing.guarantee"} for exactly-once processing guarantees.
*
@@ -443,7 +416,7 @@ public class StreamsConfig extends AbstractConfig {
private static final String COMMIT_INTERVAL_MS_DOC = "The frequency in milliseconds with which to commit processing progress." +
" For at-least-once processing, committing means to save the position (ie, offsets) of the processor." +
" For exactly-once processing, it means to commit the transaction which includes to save the position and to make the committed data in the output topic visible to consumers with isolation level read_committed." +
- " (Note, if processing.guarantee is set to " + EXACTLY_ONCE_V2 + ", " + EXACTLY_ONCE + ",the default value is " + EOS_DEFAULT_COMMIT_INTERVAL_MS + "," +
+ " (Note, if processing.guarantee is set to " + EXACTLY_ONCE_V2 + ", the default value is " + EOS_DEFAULT_COMMIT_INTERVAL_MS + "," +
" otherwise the default value is " + DEFAULT_COMMIT_INTERVAL_MS + ".";
/** {@code repartition.purge.interval.ms} */
@@ -584,8 +557,6 @@ public class StreamsConfig extends AbstractConfig {
private static final String PROCESSING_GUARANTEE_DOC = "The processing guarantee that should be used. " +
"Possible values are " + AT_LEAST_ONCE + " (default) " +
"and " + EXACTLY_ONCE_V2 + " (requires brokers version 2.5 or higher). " +
- "Deprecated options are " + EXACTLY_ONCE + " (requires brokers version 0.11.0 or higher) " +
- "and " + EXACTLY_ONCE_BETA + " (requires brokers version 2.5 or higher). " +
"Note that exactly-once processing requires a cluster of at least three brokers by default what is the " +
"recommended setting for production; for development you can change this, by adjusting broker setting " +
"transaction.state.log.replication.factor and transaction.state.log.min.isr.";
@@ -816,7 +787,7 @@ public class StreamsConfig extends AbstractConfig {
.define(PROCESSING_GUARANTEE_CONFIG,
Type.STRING,
AT_LEAST_ONCE,
- in(AT_LEAST_ONCE, EXACTLY_ONCE, EXACTLY_ONCE_BETA, EXACTLY_ONCE_V2),
+ (name, value) -> validateProcessingConfiguration((String) value),
Importance.MEDIUM,
PROCESSING_GUARANTEE_DOC)
.define(RACK_AWARE_ASSIGNMENT_TAGS_CONFIG,
@@ -1247,17 +1218,6 @@ protected StreamsConfig(final Map props,
super(CONFIG, props, doLog);
eosEnabled = StreamsConfigUtils.eosEnabled(this);
- final String processingModeConfig = getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG);
- if (processingModeConfig.equals(EXACTLY_ONCE)) {
- log.warn("Configuration parameter `{}` is deprecated and will be removed in the 4.0.0 release. " +
- "Please use `{}` instead. Note that this requires broker version 2.5+ so you should prepare "
- + "to upgrade your brokers if necessary.", EXACTLY_ONCE, EXACTLY_ONCE_V2);
- }
- if (processingModeConfig.equals(EXACTLY_ONCE_BETA)) {
- log.warn("Configuration parameter `{}` is deprecated and will be removed in the 4.0.0 release. " +
- "Please use `{}` instead.", EXACTLY_ONCE_BETA, EXACTLY_ONCE_V2);
- }
-
if (props.containsKey(RETRIES_CONFIG)) {
log.warn("Configuration parameter `{}` is deprecated and will be removed in the 4.0.0 release.", RETRIES_CONFIG);
}
@@ -1331,6 +1291,21 @@ private void validateRackAwarenessConfiguration() {
});
}
+ private static void validateProcessingConfiguration(final String processingModeConfig) {
+ if (processingModeConfig.equals("exactly_once")) {
+ throw new ConfigException(String.format("Configuration parameter `exactly_once` was removed in the 4.0.0 release. " +
+ "Please use `%s` instead. Refer to the Kafka Streams upgrade guide on how to upgrade your application " +
+ "to use the new parameter. Note that this requires broker version 2.5+ so you should prepare "
+ + "to upgrade your brokers if necessary.", EXACTLY_ONCE_V2));
+ }
+ if (processingModeConfig.equals("exactly_once_beta")) {
+ throw new ConfigException(String.format("Configuration parameter `exactly_once_beta` was removed in the 4.0.0 release. " +
+ "Please use `%s` instead, which is the new name for the same processing semantics.",
+ EXACTLY_ONCE_V2));
+ }
+ in(AT_LEAST_ONCE, EXACTLY_ONCE_V2).ensureValid(PROCESSING_GUARANTEE_CONFIG, processingModeConfig);
+ }
+
private Map getCommonConsumerConfigs() {
final Map clientProvidedProps = getClientPropsWithPrefix(CONSUMER_PREFIX, ConsumerConfig.configNames());
@@ -1580,11 +1555,6 @@ public Map getProducerConfigs(final String clientId) {
props.putAll(getClientCustomProps());
props.putAll(clientProvidedProps);
- // When using EOS alpha, stream should auto-downgrade the transactional commit protocol to be compatible with older brokers.
- if (StreamsConfigUtils.processingMode(this) == StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_ALPHA) {
- props.put("internal.auto.downgrade.txn.commit", true);
- }
-
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, originals().get(BOOTSTRAP_SERVERS_CONFIG));
// add client id with stream client id prefix
props.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId);
diff --git a/streams/src/main/java/org/apache/kafka/streams/internals/StreamsConfigUtils.java b/streams/src/main/java/org/apache/kafka/streams/internals/StreamsConfigUtils.java
index e271a42ab891..7733942089e7 100644
--- a/streams/src/main/java/org/apache/kafka/streams/internals/StreamsConfigUtils.java
+++ b/streams/src/main/java/org/apache/kafka/streams/internals/StreamsConfigUtils.java
@@ -23,8 +23,6 @@ public class StreamsConfigUtils {
public enum ProcessingMode {
AT_LEAST_ONCE("AT_LEAST_ONCE"),
- EXACTLY_ONCE_ALPHA("EXACTLY_ONCE_ALPHA"),
-
EXACTLY_ONCE_V2("EXACTLY_ONCE_V2");
public final String name;
@@ -34,25 +32,17 @@ public enum ProcessingMode {
}
}
- @SuppressWarnings("deprecation")
public static ProcessingMode processingMode(final StreamsConfig config) {
- if (StreamsConfig.EXACTLY_ONCE.equals(config.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG))) {
- return ProcessingMode.EXACTLY_ONCE_ALPHA;
- } else if (StreamsConfig.EXACTLY_ONCE_BETA.equals(config.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG))) {
- return ProcessingMode.EXACTLY_ONCE_V2;
- } else if (StreamsConfig.EXACTLY_ONCE_V2.equals(config.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG))) {
+ if (StreamsConfig.EXACTLY_ONCE_V2.equals(config.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG))) {
return ProcessingMode.EXACTLY_ONCE_V2;
} else {
return ProcessingMode.AT_LEAST_ONCE;
}
}
- @SuppressWarnings("deprecation")
public static String processingModeString(final ProcessingMode processingMode) {
if (processingMode == ProcessingMode.EXACTLY_ONCE_V2) {
return StreamsConfig.EXACTLY_ONCE_V2;
- } else if (processingMode == ProcessingMode.EXACTLY_ONCE_ALPHA) {
- return StreamsConfig.EXACTLY_ONCE;
} else {
return StreamsConfig.AT_LEAST_ONCE;
}
@@ -63,7 +53,6 @@ public static boolean eosEnabled(final StreamsConfig config) {
}
public static boolean eosEnabled(final ProcessingMode processingMode) {
- return processingMode == ProcessingMode.EXACTLY_ONCE_ALPHA ||
- processingMode == ProcessingMode.EXACTLY_ONCE_V2;
+ return processingMode == ProcessingMode.EXACTLY_ONCE_V2;
}
}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java
index d28d0d4444c7..a7bfae095003 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java
@@ -36,18 +36,15 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
-import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_ALPHA;
import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_V2;
import static org.apache.kafka.streams.internals.StreamsConfigUtils.eosEnabled;
import static org.apache.kafka.streams.internals.StreamsConfigUtils.processingMode;
-import static org.apache.kafka.streams.processor.internals.ClientUtils.getTaskProducerClientId;
import static org.apache.kafka.streams.processor.internals.ClientUtils.getThreadProducerClientId;
class ActiveTaskCreator {
@@ -58,12 +55,10 @@ class ActiveTaskCreator {
private final ChangelogReader storeChangelogReader;
private final ThreadCache cache;
private final Time time;
- private final KafkaClientSupplier clientSupplier;
private final String threadId;
private final Logger log;
private final Sensor createTaskSensor;
private final StreamsProducer threadProducer;
- private final Map taskProducers;
private final ProcessingMode processingMode;
ActiveTaskCreator(final TopologyMetadata topologyMetadata,
@@ -84,50 +79,30 @@ class ActiveTaskCreator {
this.storeChangelogReader = storeChangelogReader;
this.cache = cache;
this.time = time;
- this.clientSupplier = clientSupplier;
this.threadId = threadId;
this.log = log;
createTaskSensor = ThreadMetrics.createTaskSensor(threadId, streamsMetrics);
processingMode = processingMode(applicationConfig);
- if (processingMode == EXACTLY_ONCE_ALPHA) {
- threadProducer = null;
- taskProducers = new HashMap<>();
- } else { // non-eos and eos-v2
- log.info("Creating thread producer client");
+ log.info("Creating thread producer client");
- final String threadIdPrefix = String.format("stream-thread [%s] ", Thread.currentThread().getName());
- final LogContext logContext = new LogContext(threadIdPrefix);
+ final String threadIdPrefix = String.format("stream-thread [%s] ", Thread.currentThread().getName());
+ final LogContext logContext = new LogContext(threadIdPrefix);
- threadProducer = new StreamsProducer(
- applicationConfig,
- threadId,
- clientSupplier,
- null,
- processId,
- logContext,
- time);
- taskProducers = Collections.emptyMap();
- }
+ threadProducer = new StreamsProducer(
+ applicationConfig,
+ threadId,
+ clientSupplier,
+ processId,
+ logContext,
+ time);
}
public void reInitializeThreadProducer() {
threadProducer.resetProducer();
}
- StreamsProducer streamsProducerForTask(final TaskId taskId) {
- if (processingMode != EXACTLY_ONCE_ALPHA) {
- throw new IllegalStateException("Expected EXACTLY_ONCE to be enabled, but the processing mode was " + processingMode);
- }
-
- final StreamsProducer taskProducer = taskProducers.get(taskId);
- if (taskProducer == null) {
- throw new IllegalStateException("Unknown TaskId: " + taskId);
- }
- return taskProducer;
- }
-
StreamsProducer threadProducer() {
if (processingMode != EXACTLY_ONCE_V2) {
throw new IllegalStateException("Expected EXACTLY_ONCE_V2 to be enabled, but the processing mode was " + processingMode);
@@ -183,27 +158,10 @@ public Collection createTasks(final Consumer consumer,
private RecordCollector createRecordCollector(final TaskId taskId,
final LogContext logContext,
final ProcessorTopology topology) {
- final StreamsProducer streamsProducer;
- if (processingMode == ProcessingMode.EXACTLY_ONCE_ALPHA) {
- log.info("Creating producer client for task {}", taskId);
- streamsProducer = new StreamsProducer(
- applicationConfig,
- threadId,
- clientSupplier,
- taskId,
- null,
- logContext,
- time
- );
- taskProducers.put(taskId, streamsProducer);
- } else {
- streamsProducer = threadProducer;
- }
-
return new RecordCollectorImpl(
logContext,
taskId,
- streamsProducer,
+ threadProducer,
applicationConfig.defaultProductionExceptionHandler(),
streamsMetrics,
topology
@@ -278,45 +236,22 @@ private StreamTask createActiveTask(final TaskId taskId,
}
void closeThreadProducerIfNeeded() {
- if (threadProducer != null) {
- try {
- threadProducer.close();
- } catch (final RuntimeException e) {
- throw new StreamsException("Thread producer encounter error trying to close.", e);
- }
- }
- }
-
- void closeAndRemoveTaskProducerIfNeeded(final TaskId id) {
- final StreamsProducer taskProducer = taskProducers.remove(id);
- if (taskProducer != null) {
- try {
- taskProducer.close();
- } catch (final RuntimeException e) {
- throw new StreamsException("[" + id + "] task producer encounter error trying to close.", e, id);
- }
+ try {
+ threadProducer.close();
+ } catch (final RuntimeException e) {
+ throw new StreamsException("Thread producer encounter error trying to close.", e);
}
}
Map producerMetrics() {
- // When EOS is turned on, each task will have its own producer client
- // and the producer object passed in here will be null. We would then iterate through
- // all the active tasks and add their metrics to the output metrics map.
- final Collection producers = threadProducer != null ?
- Collections.singleton(threadProducer) :
- taskProducers.values();
- return ClientUtils.producerMetrics(producers);
+ return threadProducer.metrics()
+ .entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> (Metric) e.getValue()));
}
Set producerClientIds() {
- if (threadProducer != null) {
- return Collections.singleton(getThreadProducerClientId(threadId));
- } else {
- return taskProducers.keySet()
- .stream()
- .map(taskId -> getTaskProducerClientId(threadId, taskId))
- .collect(Collectors.toSet());
- }
+ return Collections.singleton(getThreadProducerClientId(threadId));
}
private LogContext getLogContext(final TaskId taskId) {
@@ -326,11 +261,6 @@ private LogContext getLogContext(final TaskId taskId) {
}
public double totalProducerBlockedTime() {
- if (threadProducer != null) {
- return threadProducer.totalBlockedTime();
- }
- return taskProducers.values().stream()
- .mapToDouble(StreamsProducer::totalBlockedTime)
- .sum();
+ return threadProducer.totalBlockedTime();
}
}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ClientUtils.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ClientUtils.java
index 1177e29d8259..8f54673d940e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ClientUtils.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ClientUtils.java
@@ -34,7 +34,6 @@
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.errors.StreamsException;
-import org.apache.kafka.streams.processor.TaskId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,10 +79,6 @@ public static String getThreadProducerClientId(final String threadClientId) {
return threadClientId + "-producer";
}
- public static String getTaskProducerClientId(final String threadClientId, final TaskId taskId) {
- return threadClientId + "-" + taskId + "-producer";
- }
-
public static Map consumerMetrics(final Consumer mainConsumer,
final Consumer restoreConsumer) {
final Map consumerMetrics = mainConsumer.metrics();
@@ -99,17 +94,6 @@ public static Map adminClientMetrics(final Admin adminClient
return new LinkedHashMap<>(adminClientMetrics);
}
- public static Map producerMetrics(final Collection producers) {
- final Map result = new LinkedHashMap<>();
- for (final StreamsProducer producer : producers) {
- final Map producerMetrics = producer.metrics();
- if (producerMetrics != null) {
- result.putAll(producerMetrics);
- }
- }
- return result;
- }
-
/**
* @throws StreamsException if the consumer throws an exception
* @throws org.apache.kafka.common.errors.TimeoutException if the request times out
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 0bc19cefc30b..61743bba0e2e 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -583,7 +583,6 @@ public void run() {
* @throws IllegalStateException If store gets registered after initialized is already finished
* @throws StreamsException if the store's change log does not contain the partition
*/
- @SuppressWarnings("deprecation") // Needed to include StreamsConfig.EXACTLY_ONCE_BETA in error log for UnsupportedVersionException
boolean runLoop() {
subscribeConsumer();
@@ -632,9 +631,9 @@ boolean runLoop() {
errorMessage.startsWith("Broker unexpectedly doesn't support requireStable flag on version ")) {
log.error("Shutting down because the Kafka cluster seems to be on a too old version. " +
- "Setting {}=\"{}\"/\"{}\" requires broker version 2.5 or higher.",
- StreamsConfig.PROCESSING_GUARANTEE_CONFIG,
- StreamsConfig.EXACTLY_ONCE_V2, StreamsConfig.EXACTLY_ONCE_BETA);
+ "Setting {}=\"{}\" requires broker version 2.5 or higher.",
+ StreamsConfig.PROCESSING_GUARANTEE_CONFIG,
+ StreamsConfig.EXACTLY_ONCE_V2);
}
failedStreamThreadSensor.record();
this.streamsUncaughtExceptionHandler.accept(new StreamsException(e), false);
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsProducer.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsProducer.java
index a1b68ff79085..8365d94f6c20 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsProducer.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsProducer.java
@@ -43,7 +43,6 @@
import org.apache.kafka.streams.errors.TaskMigratedException;
import org.apache.kafka.streams.internals.StreamsConfigUtils;
import org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode;
-import org.apache.kafka.streams.processor.TaskId;
import org.slf4j.Logger;
import java.util.List;
@@ -52,8 +51,6 @@
import java.util.UUID;
import java.util.concurrent.Future;
-import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_V2;
-import static org.apache.kafka.streams.processor.internals.ClientUtils.getTaskProducerClientId;
import static org.apache.kafka.streams.processor.internals.ClientUtils.getThreadProducerClientId;
/**
@@ -81,7 +78,6 @@ public class StreamsProducer {
public StreamsProducer(final StreamsConfig config,
final String threadId,
final KafkaClientSupplier clientSupplier,
- final TaskId taskId,
final UUID processId,
final LogContext logContext,
final Time time) {
@@ -102,21 +98,6 @@ public StreamsProducer(final StreamsConfig config,
break;
}
- case EXACTLY_ONCE_ALPHA: {
- producerConfigs = config.getProducerConfigs(
- getTaskProducerClientId(
- threadId,
- Objects.requireNonNull(taskId, "taskId cannot be null for exactly-once alpha")
- )
- );
-
- final String applicationId = config.getString(StreamsConfig.APPLICATION_ID_CONFIG);
- producerConfigs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, applicationId + "-" + taskId);
-
- eosV2ProducerConfigs = null;
-
- break;
- }
case EXACTLY_ONCE_V2: {
producerConfigs = config.getProducerConfigs(getThreadProducerClientId(threadId));
@@ -182,7 +163,7 @@ void initTransaction() {
}
public void resetProducer() {
- if (processingMode != EXACTLY_ONCE_V2) {
+ if (!eosEnabled()) {
throw new IllegalStateException("Expected eos-v2 to be enabled, but the processing mode was " + processingMode);
}
@@ -290,10 +271,7 @@ protected void commitTransaction(final Map of
}
maybeBeginTransaction();
try {
- // EOS-v2 assumes brokers are on version 2.5+ and thus can understand the full set of consumer group metadata
- // Thus if we are using EOS-v1 and can't make this assumption, we must downgrade the request to include only the group id metadata
- final ConsumerGroupMetadata maybeDowngradedGroupMetadata = processingMode == EXACTLY_ONCE_V2 ? consumerGroupMetadata : new ConsumerGroupMetadata(consumerGroupMetadata.groupId());
- producer.sendOffsetsToTransaction(offsets, maybeDowngradedGroupMetadata);
+ producer.sendOffsetsToTransaction(offsets, consumerGroupMetadata);
producer.commitTransaction();
transactionInFlight = false;
} catch (final ProducerFencedException | InvalidProducerEpochException | CommitFailedException error) {
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskExecutor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskExecutor.java
index b839c22f5aa0..0a0e04145ab4 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskExecutor.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskExecutor.java
@@ -37,7 +37,6 @@
import java.util.stream.Collectors;
import org.slf4j.Logger;
-import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_ALPHA;
import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_V2;
/**
@@ -176,64 +175,47 @@ void commitOffsetsOrTransaction(final Map corruptedTasks = new HashSet<>();
if (!offsetsPerTask.isEmpty()) {
- if (executionMetadata.processingMode() == EXACTLY_ONCE_ALPHA) {
- for (final Map.Entry> taskToCommit : offsetsPerTask.entrySet()) {
- final Task task = taskToCommit.getKey();
- try {
- taskManager.streamsProducerForTask(task.id())
- .commitTransaction(taskToCommit.getValue(), taskManager.mainConsumer().groupMetadata());
- updateTaskCommitMetadata(taskToCommit.getValue());
- } catch (final TimeoutException timeoutException) {
- log.error(
- String.format("Committing task %s failed.", task.id()),
- timeoutException
- );
- corruptedTasks.add(task.id());
- }
+ final Map allOffsets = offsetsPerTask.values().stream()
+ .flatMap(e -> e.entrySet().stream()).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ if (executionMetadata.processingMode() == EXACTLY_ONCE_V2) {
+ try {
+ taskManager.threadProducer().commitTransaction(allOffsets, taskManager.mainConsumer().groupMetadata());
+ updateTaskCommitMetadata(allOffsets);
+ } catch (final TimeoutException timeoutException) {
+ log.error(
+ String.format("Committing task(s) %s failed.",
+ offsetsPerTask
+ .keySet()
+ .stream()
+ .map(t -> t.id().toString())
+ .collect(Collectors.joining(", "))),
+ timeoutException
+ );
+ offsetsPerTask
+ .keySet()
+ .forEach(task -> corruptedTasks.add(task.id()));
}
} else {
- final Map allOffsets = offsetsPerTask.values().stream()
- .flatMap(e -> e.entrySet().stream()).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-
- if (executionMetadata.processingMode() == EXACTLY_ONCE_V2) {
- try {
- taskManager.threadProducer().commitTransaction(allOffsets, taskManager.mainConsumer().groupMetadata());
- updateTaskCommitMetadata(allOffsets);
- } catch (final TimeoutException timeoutException) {
- log.error(
- String.format("Committing task(s) %s failed.",
- offsetsPerTask
- .keySet()
- .stream()
- .map(t -> t.id().toString())
- .collect(Collectors.joining(", "))),
- timeoutException
- );
- offsetsPerTask
- .keySet()
- .forEach(task -> corruptedTasks.add(task.id()));
- }
- } else {
- try {
- taskManager.mainConsumer().commitSync(allOffsets);
- updateTaskCommitMetadata(allOffsets);
- } catch (final CommitFailedException error) {
- throw new TaskMigratedException("Consumer committing offsets failed, " +
- "indicating the corresponding thread is no longer part of the group", error);
- } catch (final TimeoutException timeoutException) {
- log.error(
- String.format("Committing task(s) %s failed.",
- offsetsPerTask
- .keySet()
- .stream()
- .map(t -> t.id().toString())
- .collect(Collectors.joining(", "))),
- timeoutException
- );
- throw timeoutException;
- } catch (final KafkaException error) {
- throw new StreamsException("Error encountered committing offsets via consumer", error);
- }
+ try {
+ taskManager.mainConsumer().commitSync(allOffsets);
+ updateTaskCommitMetadata(allOffsets);
+ } catch (final CommitFailedException error) {
+ throw new TaskMigratedException("Consumer committing offsets failed, " +
+ "indicating the corresponding thread is no longer part of the group", error);
+ } catch (final TimeoutException timeoutException) {
+ log.error(
+ String.format("Committing task(s) %s failed.",
+ offsetsPerTask
+ .keySet()
+ .stream()
+ .map(t -> t.id().toString())
+ .collect(Collectors.joining(", "))),
+ timeoutException
+ );
+ throw timeoutException;
+ } catch (final KafkaException error) {
+ throw new StreamsException("Error encountered committing offsets via consumer", error);
}
}
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
index 31eba40ae617..10e66ffc0526 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java
@@ -152,10 +152,6 @@ Consumer mainConsumer() {
return mainConsumer;
}
- StreamsProducer streamsProducerForTask(final TaskId taskId) {
- return activeTaskCreator.streamsProducerForTask(taskId);
- }
-
StreamsProducer threadProducer() {
return activeTaskCreator.threadProducer();
}
@@ -617,9 +613,7 @@ private Map closeAndRecycleTasks(final Map partitions) {
- final StandbyTask standbyTask = standbyTaskCreator.createStandbyTaskFromActive(activeTask, partitions);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(activeTask.id());
- return standbyTask;
+ return standbyTaskCreator.createStandbyTaskFromActive(activeTask, partitions);
}
private StreamTask convertStandbyToActive(final StandbyTask standbyTask, final Set partitions) {
@@ -743,9 +737,6 @@ private void closeTaskClean(final Task task,
try {
task.suspend();
task.closeClean();
- if (task.isActive()) {
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(task.id());
- }
} catch (final RuntimeException e) {
final String uncleanMessage = String.format("Failed to close task %s cleanly. " +
"Attempting to close remaining tasks before re-throwing:", task.id());
@@ -1195,10 +1186,6 @@ private void closeTaskDirty(final Task task, final boolean removeFromTasksRegist
if (removeFromTasksRegistry) {
tasks.removeTask(task);
}
-
- if (task.isActive()) {
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(task.id());
- }
} catch (final RuntimeException swallow) {
log.error("Error removing dirty task {}: {}", task.id(), swallow.getMessage());
}
@@ -1207,9 +1194,6 @@ private void closeTaskDirty(final Task task, final boolean removeFromTasksRegist
private void closeTaskClean(final Task task) {
task.closeClean();
tasks.removeTask(task);
- if (task.isActive()) {
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(task.id());
- }
}
void shutdown(final boolean clean) {
@@ -1281,14 +1265,6 @@ private void closeFailedTasksFromStateUpdater() {
}
task.closeDirty();
-
- try {
- if (task.isActive()) {
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(task.id());
- }
- } catch (final RuntimeException swallow) {
- log.error("Error closing dirty task {}: {}", task.id(), swallow.getMessage());
- }
}
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
index 435dd249f2f6..47f636b903ac 100644
--- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java
@@ -55,8 +55,6 @@
import static org.apache.kafka.common.utils.Utils.mkSet;
import static org.apache.kafka.streams.StreamsConfig.AT_LEAST_ONCE;
import static org.apache.kafka.streams.StreamsConfig.DEFAULT_DSL_STORE_CONFIG;
-import static org.apache.kafka.streams.StreamsConfig.EXACTLY_ONCE;
-import static org.apache.kafka.streams.StreamsConfig.EXACTLY_ONCE_BETA;
import static org.apache.kafka.streams.StreamsConfig.EXACTLY_ONCE_V2;
import static org.apache.kafka.streams.StreamsConfig.MAX_RACK_AWARE_ASSIGNMENT_TAG_KEY_LENGTH;
import static org.apache.kafka.streams.StreamsConfig.MAX_RACK_AWARE_ASSIGNMENT_TAG_VALUE_LENGTH;
@@ -409,18 +407,10 @@ public void shouldOverrideStreamsDefaultProducerConfigs() {
assertEquals("30000", producerConfigs.get(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldThrowIfTransactionTimeoutSmallerThanCommitIntervalForEOSAlpha() {
- assertThrows(IllegalArgumentException.class,
- () -> testTransactionTimeoutSmallerThanCommitInterval(EXACTLY_ONCE));
- }
-
- @SuppressWarnings("deprecation")
@Test
- public void shouldThrowIfTransactionTimeoutSmallerThanCommitIntervalForEOSBeta() {
+ public void shouldThrowIfTransactionTimeoutSmallerThanCommitIntervalForEosV2() {
assertThrows(IllegalArgumentException.class,
- () -> testTransactionTimeoutSmallerThanCommitInterval(EXACTLY_ONCE_BETA));
+ () -> testTransactionTimeoutSmallerThanCommitInterval(EXACTLY_ONCE_V2));
}
@Test
@@ -529,24 +519,6 @@ public void shouldNotSetInternalThrowOnFetchStableOffsetUnsupportedConfigToFalse
assertThat(consumerConfigs.get("internal.throw.on.fetch.stable.offset.unsupported"), is(nullValue()));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotSetInternalThrowOnFetchStableOffsetUnsupportedConfigToFalseInConsumerForEosAlpha() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- final StreamsConfig streamsConfig = new StreamsConfig(props);
- final Map consumerConfigs = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
- assertThat(consumerConfigs.get("internal.throw.on.fetch.stable.offset.unsupported"), is(nullValue()));
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotSetInternalThrowOnFetchStableOffsetUnsupportedConfigToFalseInConsumerForEosBeta() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- final StreamsConfig streamsConfig = new StreamsConfig(props);
- final Map consumerConfigs = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
- assertThat(consumerConfigs.get("internal.throw.on.fetch.stable.offset.unsupported"), is(true));
- }
-
@Test
public void shouldNotSetInternalThrowOnFetchStableOffsetUnsupportedConfigToFalseInConsumerForEosV2() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
@@ -561,24 +533,6 @@ public void shouldNotSetInternalAutoDowngradeTxnCommitToTrueInProducerForEosDisa
assertThat(producerConfigs.get("internal.auto.downgrade.txn.commit"), is(nullValue()));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldSetInternalAutoDowngradeTxnCommitToTrueInProducerForEosAlpha() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- final StreamsConfig streamsConfig = new StreamsConfig(props);
- final Map producerConfigs = streamsConfig.getProducerConfigs(clientId);
- assertThat(producerConfigs.get("internal.auto.downgrade.txn.commit"), is(true));
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotSetInternalAutoDowngradeTxnCommitToTrueInProducerForEosBeta() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- final StreamsConfig streamsConfig = new StreamsConfig(props);
- final Map producerConfigs = streamsConfig.getProducerConfigs(clientId);
- assertThat(producerConfigs.get("internal.auto.downgrade.txn.commit"), is(nullValue()));
- }
-
@Test
public void shouldNotSetInternalAutoDowngradeTxnCommitToTrueInProducerForEosV2() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
@@ -594,20 +548,6 @@ public void shouldAcceptAtLeastOnce() {
new StreamsConfig(props);
}
- @Test
- public void shouldAcceptExactlyOnce() {
- // don't use `StreamsConfig.EXACLTY_ONCE` to actually do a useful test
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "exactly_once");
- new StreamsConfig(props);
- }
-
- @Test
- public void shouldAcceptExactlyOnceBeta() {
- // don't use `StreamsConfig.EXACLTY_ONCE_BETA` to actually do a useful test
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "exactly_once_beta");
- new StreamsConfig(props);
- }
-
@Test
public void shouldThrowExceptionIfNotAtLeastOnceOrExactlyOnce() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "bad_value");
@@ -638,27 +578,9 @@ public void shouldThrowIfBuiltInMetricsVersionInvalid() {
);
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosEnabled();
- }
-
@Test
public void shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosEnabled();
- }
-
- private void shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosEnabled() {
props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "anyValue");
final StreamsConfig streamsConfig = new StreamsConfig(props);
final Map consumerConfigs = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
@@ -679,27 +601,9 @@ public void shouldAllowSettingConsumerIsolationLevelIfEosDisabled() {
);
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosEnabled();
- }
-
@Test
public void shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosEnabled();
- }
-
- private void shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosEnabled() {
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "anyValue");
final StreamsConfig streamsConfig = new StreamsConfig(props);
final Map producerConfigs = streamsConfig.getProducerConfigs(clientId);
@@ -714,27 +618,9 @@ public void shouldAllowSettingProducerEnableIdempotenceIfEosDisabled() {
assertThat(producerConfigs.get(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG), equalTo(false));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldSetDifferentDefaultsIfEosAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldSetDifferentDefaultsIfEosEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldSetDifferentDefaultsIfEosBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldSetDifferentDefaultsIfEosEnabled();
- }
-
@Test
public void shouldSetDifferentDefaultsIfEosV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldSetDifferentDefaultsIfEosEnabled();
- }
-
- private void shouldSetDifferentDefaultsIfEosEnabled() {
final StreamsConfig streamsConfig = new StreamsConfig(props);
final Map consumerConfigs = streamsConfig.getMainConsumerConfigs(groupId, clientId, threadIdx);
@@ -750,27 +636,9 @@ private void shouldSetDifferentDefaultsIfEosEnabled() {
assertThat(streamsConfig.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG), equalTo(100L));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldOverrideUserConfigTransactionalIdIfEosAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldOverrideUserConfigTransactionalIdIfEosEnable();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldOverrideUserConfigTransactionalIdIfEosBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldOverrideUserConfigTransactionalIdIfEosEnable();
- }
-
@Test
public void shouldOverrideUserConfigTransactionalIdIfEosV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldOverrideUserConfigTransactionalIdIfEosEnable();
- }
-
- private void shouldOverrideUserConfigTransactionalIdIfEosEnable() {
props.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "user-TxId");
final StreamsConfig streamsConfig = new StreamsConfig(props);
@@ -779,27 +647,9 @@ private void shouldOverrideUserConfigTransactionalIdIfEosEnable() {
assertThat(producerConfigs.get(ProducerConfig.TRANSACTIONAL_ID_CONFIG), is(nullValue()));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotOverrideUserConfigRetriesIfExactlyAlphaOnceEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldNotOverrideUserConfigRetriesIfExactlyOnceEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotOverrideUserConfigRetriesIfExactlyBetaOnceEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldNotOverrideUserConfigRetriesIfExactlyOnceEnabled();
- }
-
@Test
public void shouldNotOverrideUserConfigRetriesIfExactlyV2OnceEnabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldNotOverrideUserConfigRetriesIfExactlyOnceEnabled();
- }
-
- private void shouldNotOverrideUserConfigRetriesIfExactlyOnceEnabled() {
final int numberOfRetries = 42;
props.put(ProducerConfig.RETRIES_CONFIG, numberOfRetries);
final StreamsConfig streamsConfig = new StreamsConfig(props);
@@ -809,27 +659,9 @@ private void shouldNotOverrideUserConfigRetriesIfExactlyOnceEnabled() {
assertThat(producerConfigs.get(ProducerConfig.RETRIES_CONFIG), equalTo(numberOfRetries));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceEnabled();
- }
-
@Test
public void shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceEnabled();
- }
-
- private void shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceEnabled() {
final long commitIntervalMs = 73L;
props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, commitIntervalMs);
final StreamsConfig streamsConfig = new StreamsConfig(props);
@@ -906,27 +738,9 @@ public void shouldSpecifyCorrectValueSerdeClassOnError() {
}
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosEnabled();
- }
-
@Test
public void shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosEnabled();
- }
-
- private void shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosEnabled() {
props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 7);
final StreamsConfig streamsConfig = new StreamsConfig(props);
try {
@@ -941,53 +755,17 @@ private void shouldThrowExceptionIfMaxInFlightRequestsGreaterThanFiveIfEosEnable
}
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldAllowToSpecifyMaxInFlightRequestsPerConnectionAsStringIfEosAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldAllowToSpecifyMaxInFlightRequestsPerConnectionAsStringIfEosEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldAllowToSpecifyMaxInFlightRequestsPerConnectionAsStringIfEosBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldAllowToSpecifyMaxInFlightRequestsPerConnectionAsStringIfEosEnabled();
- }
-
@Test
public void shouldAllowToSpecifyMaxInFlightRequestsPerConnectionAsStringIfEosV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldAllowToSpecifyMaxInFlightRequestsPerConnectionAsStringIfEosEnabled();
- }
-
- private void shouldAllowToSpecifyMaxInFlightRequestsPerConnectionAsStringIfEosEnabled() {
props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "3");
new StreamsConfig(props).getProducerConfigs(clientId);
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldThrowConfigExceptionIfMaxInFlightRequestsPerConnectionIsInvalidStringIfEosAlphaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
- shouldThrowConfigExceptionIfMaxInFlightRequestsPerConnectionIsInvalidStringIfEosEnabled();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldThrowConfigExceptionIfMaxInFlightRequestsPerConnectionIsInvalidStringIfEosBetaEnabled() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_BETA);
- shouldThrowConfigExceptionIfMaxInFlightRequestsPerConnectionIsInvalidStringIfEosEnabled();
- }
-
@Test
public void shouldThrowConfigExceptionIfMaxInFlightRequestsPerConnectionIsInvalidStringIfEosV2Enabled() {
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE_V2);
- shouldThrowConfigExceptionIfMaxInFlightRequestsPerConnectionIsInvalidStringIfEosEnabled();
- }
-
- private void shouldThrowConfigExceptionIfMaxInFlightRequestsPerConnectionIsInvalidStringIfEosEnabled() {
props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "not-a-number");
try {
@@ -1053,42 +831,28 @@ public void shouldThrowConfigExceptionWhenStoreTypeConfigNotValueInRange() {
assertThrows(ConfigException.class, () -> new StreamsConfig(props));
}
- @SuppressWarnings("deprecation")
@Test
- public void shouldLogWarningWhenEosAlphaIsUsed() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
-
- LogCaptureAppender.setClassLoggerToDebug(StreamsConfig.class);
- try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(StreamsConfig.class)) {
- new StreamsConfig(props);
-
- assertThat(
- appender.getMessages(),
- hasItem("Configuration parameter `" + StreamsConfig.EXACTLY_ONCE +
- "` is deprecated and will be removed in the 4.0.0 release. " +
- "Please use `" + StreamsConfig.EXACTLY_ONCE_V2 + "` instead. " +
- "Note that this requires broker version 2.5+ so you should prepare " +
- "to upgrade your brokers if necessary.")
- );
- }
+ public void shouldThrowWhenEosAlphaIsUsed() {
+ props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "exactly_once");
+ final ConfigException exception = assertThrows(ConfigException.class, () -> new StreamsConfig(props));
+ assertEquals(
+ "Configuration parameter `exactly_once` was removed in the 4.0.0 release. " +
+ "Please use `exactly_once_v2` instead. Refer to the Kafka Streams upgrade guide on how to upgrade your application "
+ + "to use the new parameter. Note that this requires broker version 2.5+ so you should prepare "
+ + "to upgrade your brokers if necessary.",
+ exception.getMessage()
+ );
}
- @SuppressWarnings("deprecation")
@Test
- public void shouldLogWarningWhenEosBetaIsUsed() {
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_BETA);
-
- LogCaptureAppender.setClassLoggerToDebug(StreamsConfig.class);
- try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(StreamsConfig.class)) {
- new StreamsConfig(props);
-
- assertThat(
- appender.getMessages(),
- hasItem("Configuration parameter `" + StreamsConfig.EXACTLY_ONCE_BETA +
- "` is deprecated and will be removed in the 4.0.0 release. " +
- "Please use `" + StreamsConfig.EXACTLY_ONCE_V2 + "` instead.")
- );
- }
+ public void shouldThrowWhenEosBetaIsUsed() {
+ props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "exactly_once_beta");
+ final ConfigException exception = assertThrows(ConfigException.class, () -> new StreamsConfig(props));
+ assertEquals(
+ "Configuration parameter `exactly_once_beta` was removed in the 4.0.0 release. " +
+ "Please use `exactly_once_v2` instead, which is the new name for the same processing semantics.",
+ exception.getMessage()
+ );
}
@SuppressWarnings("deprecation")
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/EOSUncleanShutdownIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/EOSUncleanShutdownIntegrationTest.java
index 718f162a18c0..395ffa2ba3d5 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/EOSUncleanShutdownIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/EOSUncleanShutdownIntegrationTest.java
@@ -41,13 +41,9 @@
import org.junit.experimental.categories.Category;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.Timeout;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
@@ -65,24 +61,11 @@
/**
* Test the unclean shutdown behavior around state store cleanup.
*/
-@RunWith(Parameterized.class)
@Category(IntegrationTest.class)
public class EOSUncleanShutdownIntegrationTest {
@Rule
public Timeout globalTimeout = Timeout.seconds(600);
- @SuppressWarnings("deprecation")
- @Parameterized.Parameters(name = "{0}")
- public static Collection data() {
- return Arrays.asList(new String[][] {
- {StreamsConfig.EXACTLY_ONCE},
- {StreamsConfig.EXACTLY_ONCE_V2}
- });
- }
-
- @Parameterized.Parameter
- public String eosConfig;
-
public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(3);
@BeforeClass
@@ -114,7 +97,7 @@ public static void closeCluster() {
public void shouldWorkWithUncleanShutdownWipeOutStateStore() throws InterruptedException {
final String appId = "shouldWorkWithUncleanShutdownWipeOutStateStore";
STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appId);
- STREAMS_CONFIG.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, eosConfig);
+ STREAMS_CONFIG.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2);
final String input = "input-topic";
cleanStateBeforeTest(CLUSTER, input);
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
index 12cb0bf9563e..b7776d0be51b 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/EosIntegrationTest.java
@@ -148,12 +148,10 @@ public static void closeCluster() {
private String stateTmpDir;
- @SuppressWarnings("deprecation")
@Parameters(name = "{0}")
public static Collection data() {
return Arrays.asList(new String[][]{
{StreamsConfig.AT_LEAST_ONCE},
- {StreamsConfig.EXACTLY_ONCE},
{StreamsConfig.EXACTLY_ONCE_V2}
});
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/EosV2UpgradeIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/EosV2UpgradeIntegrationTest.java
deleted file mode 100644
index 7f652f0c7f57..000000000000
--- a/streams/src/test/java/org/apache/kafka/streams/integration/EosV2UpgradeIntegrationTest.java
+++ /dev/null
@@ -1,1213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.streams.integration;
-
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Partitioner;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.common.Cluster;
-import org.apache.kafka.common.IsolationLevel;
-import org.apache.kafka.common.serialization.ByteArraySerializer;
-import org.apache.kafka.common.serialization.LongDeserializer;
-import org.apache.kafka.common.serialization.LongSerializer;
-import org.apache.kafka.common.serialization.Serdes;
-import org.apache.kafka.common.utils.Utils;
-import org.apache.kafka.streams.KafkaStreams;
-import org.apache.kafka.streams.KafkaStreams.State;
-import org.apache.kafka.streams.KeyValue;
-import org.apache.kafka.streams.StoreQueryParameters;
-import org.apache.kafka.streams.StreamsBuilder;
-import org.apache.kafka.streams.StreamsConfig;
-import org.apache.kafka.streams.StreamsConfig.InternalConfig;
-import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler;
-import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
-import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
-import org.apache.kafka.streams.integration.utils.IntegrationTestUtils.StableAssignmentListener;
-import org.apache.kafka.streams.kstream.KStream;
-import org.apache.kafka.streams.kstream.Transformer;
-import org.apache.kafka.streams.kstream.TransformerSupplier;
-import org.apache.kafka.streams.processor.ProcessorContext;
-import org.apache.kafka.streams.processor.internals.DefaultKafkaClientSupplier;
-import org.apache.kafka.streams.state.KeyValueIterator;
-import org.apache.kafka.streams.state.KeyValueStore;
-import org.apache.kafka.streams.state.QueryableStoreTypes;
-import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
-import org.apache.kafka.streams.state.StoreBuilder;
-import org.apache.kafka.streams.state.Stores;
-import org.apache.kafka.test.IntegrationTest;
-import org.apache.kafka.test.StreamsTestUtils;
-import org.apache.kafka.test.TestUtils;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.Timeout;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.File;
-import java.io.IOException;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import static org.apache.kafka.common.utils.Utils.mkSet;
-import static org.apache.kafka.test.TestUtils.waitForCondition;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.Assert.assertFalse;
-
-@RunWith(Parameterized.class)
-@Category({IntegrationTest.class})
-public class EosV2UpgradeIntegrationTest {
- @Rule
- public Timeout globalTimeout = Timeout.seconds(600);
-
- @Parameterized.Parameters(name = "{0}")
- public static Collection data() {
- return Arrays.asList(new Boolean[][] {
- {false},
- {true}
- });
- }
-
- @Parameterized.Parameter
- public boolean injectError;
-
- private static final int NUM_BROKERS = 3;
- private static final int MAX_POLL_INTERVAL_MS = (int) Duration.ofSeconds(100L).toMillis();
- private static final long MAX_WAIT_TIME_MS = Duration.ofMinutes(1L).toMillis();
-
- private static final List> CLOSE =
- Collections.unmodifiableList(
- Arrays.asList(
- KeyValue.pair(KafkaStreams.State.RUNNING, KafkaStreams.State.PENDING_SHUTDOWN),
- KeyValue.pair(KafkaStreams.State.PENDING_SHUTDOWN, KafkaStreams.State.NOT_RUNNING)
- )
- );
- private static final List> CRASH =
- Collections.unmodifiableList(
- Collections.singletonList(
- KeyValue.pair(State.PENDING_ERROR, State.ERROR)
- )
- );
-
- public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(
- NUM_BROKERS,
- Utils.mkProperties(Collections.singletonMap("auto.create.topics.enable", "false"))
- );
-
-
- @BeforeClass
- public static void startCluster() throws IOException {
- CLUSTER.start();
- }
-
- @AfterClass
- public static void closeCluster() {
- CLUSTER.stop();
- }
-
- private static String applicationId;
- private final static int NUM_TOPIC_PARTITIONS = 4;
- private final static String CONSUMER_GROUP_ID = "readCommitted";
- private final static String MULTI_PARTITION_INPUT_TOPIC = "multiPartitionInputTopic";
- private final static String MULTI_PARTITION_OUTPUT_TOPIC = "multiPartitionOutputTopic";
- private final static String APP_DIR_1 = "appDir1";
- private final static String APP_DIR_2 = "appDir2";
- private final static String UNEXPECTED_EXCEPTION_MSG = "Fail the test since we got an unexpected exception, or " +
- "there are too many exceptions thrown, please check standard error log for more info.";
- private final String storeName = "store";
-
- private final StableAssignmentListener assignmentListener = new StableAssignmentListener();
-
- private final AtomicBoolean errorInjectedClient1 = new AtomicBoolean(false);
- private final AtomicBoolean errorInjectedClient2 = new AtomicBoolean(false);
- private final AtomicBoolean commitErrorInjectedClient1 = new AtomicBoolean(false);
- private final AtomicBoolean commitErrorInjectedClient2 = new AtomicBoolean(false);
- private final AtomicInteger commitCounterClient1 = new AtomicInteger(-1);
- private final AtomicInteger commitCounterClient2 = new AtomicInteger(-1);
- private final AtomicInteger commitRequested = new AtomicInteger(0);
-
- private int testNumber = 0;
- private Map exceptionCounts = new HashMap() {
- {
- put(APP_DIR_1, 0);
- put(APP_DIR_2, 0);
- }
- };
-
- private volatile boolean hasUnexpectedError = false;
-
- @Before
- public void createTopics() throws Exception {
- applicationId = "appId-" + ++testNumber;
- CLUSTER.deleteTopicsAndWait(
- MULTI_PARTITION_INPUT_TOPIC,
- MULTI_PARTITION_OUTPUT_TOPIC,
- applicationId + "-" + storeName + "-changelog"
- );
-
- CLUSTER.createTopic(MULTI_PARTITION_INPUT_TOPIC, NUM_TOPIC_PARTITIONS, 1);
- CLUSTER.createTopic(MULTI_PARTITION_OUTPUT_TOPIC, NUM_TOPIC_PARTITIONS, 1);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldUpgradeFromEosAlphaToEosV2() throws Exception {
- // We use two KafkaStreams clients that we upgrade from eos-alpha to eos-V2. During the upgrade,
- // we ensure that there are pending transaction and verify that data is processed correctly.
- //
- // We either close clients cleanly (`injectError = false`) or let them crash (`injectError = true`) during
- // the upgrade. For both cases, EOS should not be violated.
- //
- // Additionally, we inject errors while one client is on eos-alpha while the other client is on eos-V2:
- // For this case, we inject the error during task commit phase, i.e., after offsets are appended to a TX,
- // and before the TX is committed. The goal is to verify that the written but uncommitted offsets are not
- // picked up, i.e., GroupCoordinator fencing works correctly.
- //
- // The commit interval is set to MAX_VALUE and the used `Processor` request commits manually so we have full
- // control when a commit actually happens. We use an input topic with 4 partitions and each task will request
- // a commit after processing 10 records.
- //
- // 1. start both clients and wait until rebalance stabilizes
- // 2. write 10 records per input topic partition and verify that the result was committed
- // 3. write 5 records per input topic partition to get pending transactions (verified via "read_uncommitted" mode)
- // - all 4 pending transactions are based on task producers
- // - we will get only 4 pending writes for one partition for the crash case as we crash processing the 5th record
- // 4. stop/crash the first client, wait until rebalance stabilizes:
- // - stop case:
- // * verify that the stopped client did commit its pending transaction during shutdown
- // * the second client will still have two pending transaction
- // - crash case:
- // * the pending transactions of the crashed client got aborted
- // * the second client will have four pending transactions
- // 5. restart the first client with eos-V2 enabled and wait until rebalance stabilizes
- // - the rebalance should result in a commit of all tasks
- // 6. write 5 record per input topic partition
- // - stop case:
- // * verify that the result was committed
- // - crash case:
- // * fail the second (i.e., eos-alpha) client during commit
- // * the eos-V2 client should not pickup the pending offsets
- // * verify uncommitted and committed result
- // 7. only for crash case:
- // 7a. restart the second client in eos-alpha mode and wait until rebalance stabilizes
- // 7b. write 10 records per input topic partition
- // * fail the first (i.e., eos-V2) client during commit
- // * the eos-alpha client should not pickup the pending offsets
- // * verify uncommitted and committed result
- // 7c. restart the first client in eos-V2 mode and wait until rebalance stabilizes
- // 8. write 5 records per input topic partition to get pending transactions (verified via "read_uncommitted" mode)
- // - 2 transaction are base on a task producer; one transaction is based on a thread producer
- // - we will get 4 pending writes for the crash case as we crash processing the 5th record
- // 9. stop/crash the second client and wait until rebalance stabilizes:
- // - stop only:
- // * verify that the stopped client did commit its pending transaction during shutdown
- // * the first client will still have one pending transaction
- // - crash case:
- // * the pending transactions of the crashed client got aborted
- // * the first client will have one pending transactions
- // 10. restart the second client with eos-V2 enabled and wait until rebalance stabilizes
- // - the rebalance should result in a commit of all tasks
- // 11. write 5 record per input topic partition and verify that the result was committed
-
- final List> stateTransitions1 = new LinkedList<>();
- KafkaStreams streams1Alpha = null;
- KafkaStreams streams1V2 = null;
- KafkaStreams streams1V2Two = null;
-
- final List> stateTransitions2 = new LinkedList<>();
- KafkaStreams streams2Alpha = null;
- KafkaStreams streams2AlphaTwo = null;
- KafkaStreams streams2V2 = null;
-
- try {
- // phase 1: start both clients
- streams1Alpha = getKafkaStreams(APP_DIR_1, StreamsConfig.EXACTLY_ONCE);
- streams1Alpha.setStateListener(
- (newState, oldState) -> stateTransitions1.add(KeyValue.pair(oldState, newState))
- );
-
- assignmentListener.prepareForRebalance();
- streams1Alpha.cleanUp();
- streams1Alpha.start();
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions1);
-
- streams2Alpha = getKafkaStreams(APP_DIR_2, StreamsConfig.EXACTLY_ONCE);
- streams2Alpha.setStateListener(
- (newState, oldState) -> stateTransitions2.add(KeyValue.pair(oldState, newState))
- );
- stateTransitions1.clear();
-
- assignmentListener.prepareForRebalance();
- streams2Alpha.cleanUp();
- streams2Alpha.start();
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions1);
- waitForRunning(stateTransitions2);
-
- // in all phases, we write comments that assume that p-0/p-1 are assigned to the first client
- // and p-2/p-3 are assigned to the second client (in reality the assignment might be different though)
-
- // phase 2: (write first batch of data)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // p-0: ---> 10 rec + C
- // p-1: ---> 10 rec + C
- // p-2: ---> 10 rec + C
- // p-3: ---> 10 rec + C
- final List> committedInputDataBeforeUpgrade =
- prepareData(0L, 10L, 0L, 1L, 2L, 3L);
- writeInputData(committedInputDataBeforeUpgrade);
-
- waitForCondition(
- () -> commitRequested.get() == 4,
- MAX_WAIT_TIME_MS,
- "SteamsTasks did not request commit."
- );
-
- final Map committedState = new HashMap<>();
- final List> expectedUncommittedResult =
- computeExpectedResult(committedInputDataBeforeUpgrade, committedState);
- verifyCommitted(expectedUncommittedResult);
-
- // phase 3: (write partial second batch of data)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // stop case:
- // p-0: 10 rec + C ---> 5 rec (pending)
- // p-1: 10 rec + C ---> 5 rec (pending)
- // p-2: 10 rec + C ---> 5 rec (pending)
- // p-3: 10 rec + C ---> 5 rec (pending)
- // crash case: (we just assumes that we inject the error for p-0; in reality it might be a different partition)
- // (we don't crash right away and write one record less)
- // p-0: 10 rec + C ---> 4 rec (pending)
- // p-1: 10 rec + C ---> 5 rec (pending)
- // p-2: 10 rec + C ---> 5 rec (pending)
- // p-3: 10 rec + C ---> 5 rec (pending)
- final Set cleanKeys = mkSet(0L, 1L, 2L, 3L);
- final Set keysFirstClientAlpha = keysFromInstance(streams1Alpha);
- final long firstFailingKeyForCrashCase = keysFirstClientAlpha.iterator().next();
- cleanKeys.remove(firstFailingKeyForCrashCase);
-
- final List> uncommittedInputDataBeforeFirstUpgrade = new LinkedList<>();
- final HashMap uncommittedState = new HashMap<>(committedState);
- if (!injectError) {
- uncommittedInputDataBeforeFirstUpgrade.addAll(
- prepareData(10L, 15L, 0L, 1L, 2L, 3L)
- );
- writeInputData(uncommittedInputDataBeforeFirstUpgrade);
-
- expectedUncommittedResult.addAll(
- computeExpectedResult(uncommittedInputDataBeforeFirstUpgrade, uncommittedState)
- );
- verifyUncommitted(expectedUncommittedResult);
- } else {
- final List> uncommittedInputDataWithoutFailingKey = new LinkedList<>();
- for (final long key : cleanKeys) {
- uncommittedInputDataWithoutFailingKey.addAll(prepareData(10L, 15L, key));
- }
- uncommittedInputDataWithoutFailingKey.addAll(
- prepareData(10L, 14L, firstFailingKeyForCrashCase)
- );
- uncommittedInputDataBeforeFirstUpgrade.addAll(uncommittedInputDataWithoutFailingKey);
- writeInputData(uncommittedInputDataWithoutFailingKey);
-
- expectedUncommittedResult.addAll(
- computeExpectedResult(uncommittedInputDataWithoutFailingKey, new HashMap<>(committedState))
- );
- verifyUncommitted(expectedUncommittedResult);
- }
-
- // phase 4: (stop first client)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // stop case: (client 1 will commit its two tasks on close())
- // p-0: 10 rec + C + 5 rec ---> C
- // p-1: 10 rec + C + 5 rec ---> C
- // p-2: 10 rec + C + 5 rec (pending)
- // p-3: 10 rec + C + 5 rec (pending)
- // crash case: (we write the last record that will trigger the crash; both TX from client 1 will be aborted
- // during fail over by client 2 and retried)
- // p-0: 10 rec + C + 4 rec + A + 5 rec (pending)
- // p-1: 10 rec + C + 5 rec + A + 5 rec (pending)
- // p-2: 10 rec + C + 5 rec (pending)
- // p-3: 10 rec + C + 5 rec (pending)
- stateTransitions2.clear();
- assignmentListener.prepareForRebalance();
-
- if (!injectError) {
- stateTransitions1.clear();
- streams1Alpha.close();
- waitForStateTransition(stateTransitions1, CLOSE);
- } else {
- errorInjectedClient1.set(true);
-
- final List> dataPotentiallyFirstFailingKey =
- prepareData(14L, 15L, firstFailingKeyForCrashCase);
- uncommittedInputDataBeforeFirstUpgrade.addAll(dataPotentiallyFirstFailingKey);
- writeInputData(dataPotentiallyFirstFailingKey);
- }
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions2);
-
- if (!injectError) {
- final List> committedInputDataDuringFirstUpgrade =
- uncommittedInputDataBeforeFirstUpgrade
- .stream()
- .filter(pair -> keysFirstClientAlpha.contains(pair.key))
- .collect(Collectors.toList());
-
- final List> expectedCommittedResult =
- computeExpectedResult(committedInputDataDuringFirstUpgrade, committedState);
- verifyCommitted(expectedCommittedResult);
- } else {
- // retrying TX
- expectedUncommittedResult.addAll(computeExpectedResult(
- uncommittedInputDataBeforeFirstUpgrade
- .stream()
- .filter(pair -> keysFirstClientAlpha.contains(pair.key))
- .collect(Collectors.toList()),
- new HashMap<>(committedState)
- ));
- verifyUncommitted(expectedUncommittedResult);
- waitForStateTransitionContains(stateTransitions1, CRASH);
-
- errorInjectedClient1.set(false);
- stateTransitions1.clear();
- streams1Alpha.close();
- assertFalse(UNEXPECTED_EXCEPTION_MSG, hasUnexpectedError);
- }
-
- // phase 5: (restart first client)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // stop case: (client 2 (alpha) will commit the two revoked task that migrate back to client 1)
- // (note: we may or may not get newly committed data, depending if the already committed tasks
- // migrate back to client 1, or different tasks)
- // (below we show the case for which we don't get newly committed data)
- // p-0: 10 rec + C + 5 rec ---> C
- // p-1: 10 rec + C + 5 rec ---> C
- // p-2: 10 rec + C + 5 rec (pending)
- // p-3: 10 rec + C + 5 rec (pending)
- // crash case: (client 2 (alpha) will commit all tasks even only two tasks are revoked and migrate back to client 1)
- // (note: because nothing was committed originally, we always get newly committed data)
- // p-0: 10 rec + C + 4 rec + A + 5 rec ---> C
- // p-1: 10 rec + C + 5 rec + A + 5 rec ---> C
- // p-2: 10 rec + C + 5 rec ---> C
- // p-3: 10 rec + C + 5 rec ---> C
- commitRequested.set(0);
- stateTransitions1.clear();
- stateTransitions2.clear();
- streams1V2 = getKafkaStreams(APP_DIR_1, StreamsConfig.EXACTLY_ONCE_V2);
- streams1V2.setStateListener((newState, oldState) -> stateTransitions1.add(KeyValue.pair(oldState, newState)));
- assignmentListener.prepareForRebalance();
- streams1V2.start();
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions1);
- waitForRunning(stateTransitions2);
-
- final Set newlyCommittedKeys;
- if (!injectError) {
- newlyCommittedKeys = keysFromInstance(streams1V2);
- newlyCommittedKeys.removeAll(keysFirstClientAlpha);
- } else {
- newlyCommittedKeys = mkSet(0L, 1L, 2L, 3L);
- }
-
- final List> expectedCommittedResultAfterRestartFirstClient = computeExpectedResult(
- uncommittedInputDataBeforeFirstUpgrade
- .stream()
- .filter(pair -> newlyCommittedKeys.contains(pair.key))
- .collect(Collectors.toList()),
- committedState
- );
- verifyCommitted(expectedCommittedResultAfterRestartFirstClient);
-
- // phase 6: (complete second batch of data; crash: let second client fail on commit)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // stop case: (both client commit regularly)
- // (depending on the task movement in phase 5, we may or may not get newly committed data;
- // we show the case for which p-2 and p-3 are newly committed below)
- // p-0: 10 rec + C + 5 rec + C ---> 5 rec + C
- // p-1: 10 rec + C + 5 rec + C ---> 5 rec + C
- // p-2: 10 rec + C + 5 rec ---> 5 rec + C
- // p-3: 10 rec + C + 5 rec ---> 5 rec + C
- // crash case: (second/alpha client fails and both TX are aborted)
- // (first/V2 client reprocessed the 10 records and commits TX)
- // p-0: 10 rec + C + 4 rec + A + 5 rec + C ---> 5 rec + C
- // p-1: 10 rec + C + 5 rec + A + 5 rec + C ---> 5 rec + C
- // p-2: 10 rec + C + 5 rec + C ---> 5 rec + A + 5 rec + C
- // p-3: 10 rec + C + 5 rec + C ---> 5 rec + A + 5 rec + C
- commitCounterClient1.set(0);
-
- if (!injectError) {
- final List> finishSecondBatch = prepareData(15L, 20L, 0L, 1L, 2L, 3L);
- writeInputData(finishSecondBatch);
-
- final List> committedInputDataDuringUpgrade = uncommittedInputDataBeforeFirstUpgrade
- .stream()
- .filter(pair -> !keysFirstClientAlpha.contains(pair.key))
- .filter(pair -> !newlyCommittedKeys.contains(pair.key))
- .collect(Collectors.toList());
- committedInputDataDuringUpgrade.addAll(
- finishSecondBatch
- );
-
- expectedUncommittedResult.addAll(
- computeExpectedResult(finishSecondBatch, uncommittedState)
- );
- final List> expectedCommittedResult =
- computeExpectedResult(committedInputDataDuringUpgrade, committedState);
- verifyCommitted(expectedCommittedResult);
- } else {
- final Set keysFirstClientV2 = keysFromInstance(streams1V2);
- final Set keysSecondClientAlpha = keysFromInstance(streams2Alpha);
-
- final List> committedInputDataAfterFirstUpgrade =
- prepareData(15L, 20L, keysFirstClientV2.toArray(new Long[0]));
- writeInputData(committedInputDataAfterFirstUpgrade);
-
- final List> expectedCommittedResultBeforeFailure =
- computeExpectedResult(committedInputDataAfterFirstUpgrade, committedState);
- verifyCommitted(expectedCommittedResultBeforeFailure);
- expectedUncommittedResult.addAll(expectedCommittedResultBeforeFailure);
-
- commitCounterClient2.set(0);
-
- final Iterator it = keysSecondClientAlpha.iterator();
- final Long otherKey = it.next();
- final Long failingKey = it.next();
-
- final List> uncommittedInputDataAfterFirstUpgrade =
- prepareData(15L, 19L, keysSecondClientAlpha.toArray(new Long[0]));
- uncommittedInputDataAfterFirstUpgrade.addAll(prepareData(19L, 20L, otherKey));
- writeInputData(uncommittedInputDataAfterFirstUpgrade);
-
- uncommittedState.putAll(committedState);
- expectedUncommittedResult.addAll(
- computeExpectedResult(uncommittedInputDataAfterFirstUpgrade, uncommittedState)
- );
- verifyUncommitted(expectedUncommittedResult);
-
- stateTransitions1.clear();
- stateTransitions2.clear();
- assignmentListener.prepareForRebalance();
-
- commitCounterClient1.set(0);
- commitErrorInjectedClient2.set(true);
-
- final List> dataFailingKey = prepareData(19L, 20L, failingKey);
- uncommittedInputDataAfterFirstUpgrade.addAll(dataFailingKey);
- writeInputData(dataFailingKey);
-
- expectedUncommittedResult.addAll(
- computeExpectedResult(dataFailingKey, uncommittedState)
- );
- verifyUncommitted(expectedUncommittedResult);
-
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
-
- waitForStateTransitionContains(stateTransitions2, CRASH);
-
- commitErrorInjectedClient2.set(false);
- stateTransitions2.clear();
- streams2Alpha.close();
- assertFalse(UNEXPECTED_EXCEPTION_MSG, hasUnexpectedError);
-
- final List> expectedCommittedResultAfterFailure =
- computeExpectedResult(uncommittedInputDataAfterFirstUpgrade, committedState);
- verifyCommitted(expectedCommittedResultAfterFailure);
- expectedUncommittedResult.addAll(expectedCommittedResultAfterFailure);
- }
-
- // 7. only for crash case:
- // 7a. restart the failed second client in eos-alpha mode and wait until rebalance stabilizes
- // 7b. write third batch of input data
- // * fail the first (i.e., eos-V2) client during commit
- // * the eos-alpha client should not pickup the pending offsets
- // * verify uncommitted and committed result
- // 7c. restart the first client in eos-V2 mode and wait until rebalance stabilizes
- //
- // crash case:
- // p-0: 10 rec + C + 4 rec + A + 5 rec + C + 5 rec + C ---> 10 rec + A + 10 rec + C
- // p-1: 10 rec + C + 5 rec + A + 5 rec + C + 5 rec + C ---> 10 rec + A + 10 rec + C
- // p-2: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C ---> 10 rec + C
- // p-3: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C ---> 10 rec + C
- if (!injectError) {
- streams2AlphaTwo = streams2Alpha;
- } else {
- // 7a restart the second client in eos-alpha mode and wait until rebalance stabilizes
- commitCounterClient1.set(0);
- commitCounterClient2.set(-1);
- stateTransitions1.clear();
- stateTransitions2.clear();
- streams2AlphaTwo = getKafkaStreams(APP_DIR_2, StreamsConfig.EXACTLY_ONCE);
- streams2AlphaTwo.setStateListener(
- (newState, oldState) -> stateTransitions2.add(KeyValue.pair(oldState, newState))
- );
- assignmentListener.prepareForRebalance();
- streams2AlphaTwo.start();
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions1);
- waitForRunning(stateTransitions2);
-
- // 7b. write third batch of input data
- final Set keysFirstClientV2 = keysFromInstance(streams1V2);
- final Set keysSecondClientAlphaTwo = keysFromInstance(streams2AlphaTwo);
-
- final List> committedInputDataBetweenUpgrades =
- prepareData(20L, 30L, keysSecondClientAlphaTwo.toArray(new Long[0]));
- writeInputData(committedInputDataBetweenUpgrades);
-
- final List> expectedCommittedResultBeforeFailure =
- computeExpectedResult(committedInputDataBetweenUpgrades, committedState);
- verifyCommitted(expectedCommittedResultBeforeFailure);
- expectedUncommittedResult.addAll(expectedCommittedResultBeforeFailure);
-
- commitCounterClient2.set(0);
-
- final Iterator it = keysFirstClientV2.iterator();
- final Long otherKey = it.next();
- final Long failingKey = it.next();
-
- final List> uncommittedInputDataBetweenUpgrade =
- prepareData(20L, 29L, keysFirstClientV2.toArray(new Long[0]));
- uncommittedInputDataBetweenUpgrade.addAll(prepareData(29L, 30L, otherKey));
- writeInputData(uncommittedInputDataBetweenUpgrade);
-
- uncommittedState.putAll(committedState);
- expectedUncommittedResult.addAll(
- computeExpectedResult(uncommittedInputDataBetweenUpgrade, uncommittedState)
- );
- verifyUncommitted(expectedUncommittedResult);
-
- stateTransitions1.clear();
- stateTransitions2.clear();
- assignmentListener.prepareForRebalance();
- commitCounterClient2.set(0);
- commitErrorInjectedClient1.set(true);
-
- final List> dataFailingKey = prepareData(29L, 30L, failingKey);
- uncommittedInputDataBetweenUpgrade.addAll(dataFailingKey);
- writeInputData(dataFailingKey);
-
- expectedUncommittedResult.addAll(
- computeExpectedResult(dataFailingKey, uncommittedState)
- );
- verifyUncommitted(expectedUncommittedResult);
-
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
-
- waitForStateTransitionContains(stateTransitions1, CRASH);
-
- commitErrorInjectedClient1.set(false);
- stateTransitions1.clear();
- streams1V2.close();
- assertFalse(UNEXPECTED_EXCEPTION_MSG, hasUnexpectedError);
-
- final List> expectedCommittedResultAfterFailure =
- computeExpectedResult(uncommittedInputDataBetweenUpgrade, committedState);
- verifyCommitted(expectedCommittedResultAfterFailure);
- expectedUncommittedResult.addAll(expectedCommittedResultAfterFailure);
-
- // 7c. restart the first client in eos-V2 mode and wait until rebalance stabilizes
- stateTransitions1.clear();
- stateTransitions2.clear();
- streams1V2Two = getKafkaStreams(APP_DIR_1, StreamsConfig.EXACTLY_ONCE_V2);
- streams1V2Two.setStateListener((newState, oldState) -> stateTransitions1.add(KeyValue.pair(oldState, newState)));
- assignmentListener.prepareForRebalance();
- streams1V2Two.start();
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions1);
- waitForRunning(stateTransitions2);
- }
-
- // phase 8: (write partial last batch of data)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // stop case:
- // p-0: 10 rec + C + 5 rec + C + 5 rec + C ---> 5 rec (pending)
- // p-1: 10 rec + C + 5 rec + C + 5 rec + C ---> 5 rec (pending)
- // p-2: 10 rec + C + 5 rec + C + 5 rec + C ---> 5 rec (pending)
- // p-3: 10 rec + C + 5 rec + C + 5 rec + C ---> 5 rec (pending)
- // crash case: (we just assumes that we inject the error for p-2; in reality it might be a different partition)
- // (we don't crash right away and write one record less)
- // p-0: 10 rec + C + 4 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C ---> 5 rec (pending)
- // p-1: 10 rec + C + 5 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C ---> 5 rec (pending)
- // p-2: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C ---> 4 rec (pending)
- // p-3: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C ---> 5 rec (pending)
- cleanKeys.addAll(mkSet(0L, 1L, 2L, 3L));
- final Set keysSecondClientAlphaTwo = keysFromInstance(streams2AlphaTwo);
- final long secondFailingKeyForCrashCase = keysSecondClientAlphaTwo.iterator().next();
- cleanKeys.remove(secondFailingKeyForCrashCase);
-
- final List> uncommittedInputDataBeforeSecondUpgrade = new LinkedList<>();
- if (!injectError) {
- uncommittedInputDataBeforeSecondUpgrade.addAll(
- prepareData(30L, 35L, 0L, 1L, 2L, 3L)
- );
- writeInputData(uncommittedInputDataBeforeSecondUpgrade);
-
- expectedUncommittedResult.addAll(
- computeExpectedResult(uncommittedInputDataBeforeSecondUpgrade, new HashMap<>(committedState))
- );
- verifyUncommitted(expectedUncommittedResult);
- } else {
- final List> uncommittedInputDataWithoutFailingKey = new LinkedList<>();
- for (final long key : cleanKeys) {
- uncommittedInputDataWithoutFailingKey.addAll(prepareData(30L, 35L, key));
- }
- uncommittedInputDataWithoutFailingKey.addAll(
- prepareData(30L, 34L, secondFailingKeyForCrashCase)
- );
- uncommittedInputDataBeforeSecondUpgrade.addAll(uncommittedInputDataWithoutFailingKey);
- writeInputData(uncommittedInputDataWithoutFailingKey);
-
- expectedUncommittedResult.addAll(
- computeExpectedResult(uncommittedInputDataWithoutFailingKey, new HashMap<>(committedState))
- );
- verifyUncommitted(expectedUncommittedResult);
- }
-
- // phase 9: (stop/crash second client)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // stop case: (client 2 (alpha) will commit its two tasks on close())
- // p-0: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec (pending)
- // p-1: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec (pending)
- // p-2: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec ---> C
- // p-3: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec ---> C
- // crash case: (we write the last record that will trigger the crash; both TX from client 2 will be aborted
- // during fail over by client 1 and retried)
- // p-0: 10 rec + C + 4 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C + 5 rec (pending)
- // p-1: 10 rec + C + 5 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C + 5 rec (pending)
- // p-2: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C + 4 rec ---> A + 5 rec (pending)
- // p-3: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C + 5 rec ---> A + 5 rec (pending)
- stateTransitions1.clear();
- assignmentListener.prepareForRebalance();
- if (!injectError) {
- stateTransitions2.clear();
- streams2AlphaTwo.close();
- waitForStateTransition(stateTransitions2, CLOSE);
- } else {
- errorInjectedClient2.set(true);
-
- final List> dataPotentiallySecondFailingKey =
- prepareData(34L, 35L, secondFailingKeyForCrashCase);
- uncommittedInputDataBeforeSecondUpgrade.addAll(dataPotentiallySecondFailingKey);
- writeInputData(dataPotentiallySecondFailingKey);
- }
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions1);
-
- if (!injectError) {
- final List> committedInputDataDuringSecondUpgrade =
- uncommittedInputDataBeforeSecondUpgrade
- .stream()
- .filter(pair -> keysSecondClientAlphaTwo.contains(pair.key))
- .collect(Collectors.toList());
-
- final List> expectedCommittedResult =
- computeExpectedResult(committedInputDataDuringSecondUpgrade, committedState);
- verifyCommitted(expectedCommittedResult);
- } else {
- // retrying TX
- expectedUncommittedResult.addAll(computeExpectedResult(
- uncommittedInputDataBeforeSecondUpgrade
- .stream()
- .filter(pair -> keysSecondClientAlphaTwo.contains(pair.key))
- .collect(Collectors.toList()),
- new HashMap<>(committedState)
- ));
- verifyUncommitted(expectedUncommittedResult);
- waitForStateTransitionContains(stateTransitions2, CRASH);
-
- errorInjectedClient2.set(false);
- stateTransitions2.clear();
- streams2AlphaTwo.close();
- assertFalse(UNEXPECTED_EXCEPTION_MSG, hasUnexpectedError);
- }
-
- // phase 10: (restart second client)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // the state below indicate the case for which the "original" tasks of client2 are migrated back to client2
- // if a task "switch" happens, we might get additional commits (omitted in the comment for brevity)
- //
- // stop case: (client 1 (V2) will commit all four tasks if at least one revoked and migrate task needs committing back to client 2)
- // p-0: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec ---> C
- // p-1: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec ---> C
- // p-2: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec + C
- // p-3: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec + C
- // crash case: (client 1 (V2) will commit all four tasks even only two are migrate back to client 2)
- // p-0: 10 rec + C + 4 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C + 5 rec ---> C
- // p-1: 10 rec + C + 5 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C + 5 rec ---> C
- // p-2: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C + 4 rec + A + 5 rec ---> C
- // p-3: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C + 5 rec + A + 5 rec ---> C
- commitRequested.set(0);
- stateTransitions1.clear();
- stateTransitions2.clear();
- streams2V2 = getKafkaStreams(APP_DIR_1, StreamsConfig.EXACTLY_ONCE_V2);
- streams2V2.setStateListener(
- (newState, oldState) -> stateTransitions2.add(KeyValue.pair(oldState, newState))
- );
- assignmentListener.prepareForRebalance();
- streams2V2.start();
- assignmentListener.waitForNextStableAssignment(MAX_WAIT_TIME_MS);
- waitForRunning(stateTransitions1);
- waitForRunning(stateTransitions2);
-
- newlyCommittedKeys.clear();
- if (!injectError) {
- newlyCommittedKeys.addAll(keysFromInstance(streams2V2));
- newlyCommittedKeys.removeAll(keysSecondClientAlphaTwo);
- } else {
- newlyCommittedKeys.addAll(mkSet(0L, 1L, 2L, 3L));
- }
-
- final List> expectedCommittedResultAfterRestartSecondClient = computeExpectedResult(
- uncommittedInputDataBeforeSecondUpgrade
- .stream()
- .filter(pair -> newlyCommittedKeys.contains(pair.key))
- .collect(Collectors.toList()),
- committedState
- );
- verifyCommitted(expectedCommittedResultAfterRestartSecondClient);
-
- // phase 11: (complete fourth batch of data)
- // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
- //
- // stop case:
- // p-0: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec + C ---> 5 rec + C
- // p-1: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec + C ---> 5 rec + C
- // p-2: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec + C ---> 5 rec + C
- // p-3: 10 rec + C + 5 rec + C + 5 rec + C + 5 rec + C ---> 5 rec + C
- // crash case: (we just assumes that we inject the error for p-2; in reality it might be a different partition)
- // p-0: 10 rec + C + 4 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C + 5 rec + C ---> 5 rec + C
- // p-1: 10 rec + C + 5 rec + A + 5 rec + C + 5 rec + C + 10 rec + A + 10 rec + C + 5 rec + C ---> 5 rec + C
- // p-2: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C + 4 rec + A + 5 rec + C ---> 5 rec + C
- // p-3: 10 rec + C + 5 rec + C + 5 rec + A + 5 rec + C + 10 rec + C + 5 rec + A + 5 rec + C ---> 5 rec + C
- commitCounterClient1.set(-1);
- commitCounterClient2.set(-1);
-
- final List> finishLastBatch =
- prepareData(35L, 40L, 0L, 1L, 2L, 3L);
- writeInputData(finishLastBatch);
-
- final Set uncommittedKeys = mkSet(0L, 1L, 2L, 3L);
- uncommittedKeys.removeAll(keysSecondClientAlphaTwo);
- uncommittedKeys.removeAll(newlyCommittedKeys);
- final List> committedInputDataDuringUpgrade = uncommittedInputDataBeforeSecondUpgrade
- .stream()
- .filter(pair -> uncommittedKeys.contains(pair.key))
- .collect(Collectors.toList());
- committedInputDataDuringUpgrade.addAll(
- finishLastBatch
- );
-
- final List> expectedCommittedResult =
- computeExpectedResult(committedInputDataDuringUpgrade, committedState);
- verifyCommitted(expectedCommittedResult);
- } finally {
- if (streams1Alpha != null) {
- streams1Alpha.close();
- }
- if (streams1V2 != null) {
- streams1V2.close();
- }
- if (streams1V2Two != null) {
- streams1V2Two.close();
- }
- if (streams2Alpha != null) {
- streams2Alpha.close();
- }
- if (streams2AlphaTwo != null) {
- streams2AlphaTwo.close();
- }
- if (streams2V2 != null) {
- streams2V2.close();
- }
- }
- }
-
- @SuppressWarnings("deprecation")
- private KafkaStreams getKafkaStreams(final String appDir,
- final String processingGuarantee) {
- final StreamsBuilder builder = new StreamsBuilder();
-
- final String[] storeNames = new String[] {storeName};
- final StoreBuilder> storeBuilder = Stores
- .keyValueStoreBuilder(Stores.persistentKeyValueStore(storeName), Serdes.Long(), Serdes.Long())
- .withCachingEnabled();
-
- builder.addStateStore(storeBuilder);
-
- final KStream input = builder.stream(MULTI_PARTITION_INPUT_TOPIC);
- input.transform(new TransformerSupplier>() {
- @Override
- public Transformer> get() {
- return new Transformer>() {
- ProcessorContext context;
- KeyValueStore state = null;
- AtomicBoolean crash;
- AtomicInteger sharedCommit;
-
- @Override
- public void init(final ProcessorContext context) {
- this.context = context;
- state = context.getStateStore(storeName);
- final String clientId = context.appConfigs().get(StreamsConfig.CLIENT_ID_CONFIG).toString();
- if (APP_DIR_1.equals(clientId)) {
- crash = errorInjectedClient1;
- sharedCommit = commitCounterClient1;
- } else {
- crash = errorInjectedClient2;
- sharedCommit = commitCounterClient2;
- }
- }
-
- @Override
- public KeyValue transform(final Long key, final Long value) {
- if ((value + 1) % 10 == 0) {
- if (sharedCommit.get() < 0 ||
- sharedCommit.incrementAndGet() == 2) {
-
- context.commit();
- }
- commitRequested.incrementAndGet();
- }
-
- Long sum = state.get(key);
- if (sum == null) {
- sum = value;
- } else {
- sum += value;
- }
- state.put(key, sum);
- state.flush();
-
- if (value % 10 == 4 && // potentially crash when processing 5th, 15th, or 25th record (etc.)
- crash != null && crash.compareAndSet(true, false)) {
- // only crash a single task
- throw new RuntimeException("Injected test exception.");
- }
-
- return new KeyValue<>(key, state.get(key));
- }
-
- @Override
- public void close() {}
- };
- } }, storeNames)
- .to(MULTI_PARTITION_OUTPUT_TOPIC);
-
- final Properties properties = new Properties();
- properties.put(StreamsConfig.CLIENT_ID_CONFIG, appDir);
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, processingGuarantee);
- final long commitInterval = Duration.ofMinutes(1L).toMillis();
- properties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, commitInterval);
- properties.put(StreamsConfig.consumerPrefix(ConsumerConfig.METADATA_MAX_AGE_CONFIG), Duration.ofSeconds(1L).toMillis());
- properties.put(StreamsConfig.consumerPrefix(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), "earliest");
- properties.put(StreamsConfig.consumerPrefix(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), (int) Duration.ofSeconds(5L).toMillis());
- properties.put(StreamsConfig.consumerPrefix(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), (int) Duration.ofSeconds(5L).minusMillis(1L).toMillis());
- properties.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), MAX_POLL_INTERVAL_MS);
- properties.put(StreamsConfig.producerPrefix(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG), (int) commitInterval);
- properties.put(StreamsConfig.producerPrefix(ProducerConfig.PARTITIONER_CLASS_CONFIG), KeyPartitioner.class);
- properties.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
- properties.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath() + File.separator + appDir);
- properties.put(InternalConfig.ASSIGNMENT_LISTENER, assignmentListener);
-
- final Properties config = StreamsTestUtils.getStreamsConfig(
- applicationId,
- CLUSTER.bootstrapServers(),
- Serdes.LongSerde.class.getName(),
- Serdes.LongSerde.class.getName(),
- properties
- );
-
- final KafkaStreams streams = new KafkaStreams(builder.build(), config, new TestKafkaClientSupplier());
- streams.setUncaughtExceptionHandler(e -> {
- if (!injectError) {
- // we don't expect any exception thrown in stop case
- e.printStackTrace(System.err);
- hasUnexpectedError = true;
- } else {
- int exceptionCount = (int) exceptionCounts.get(appDir);
- // should only have our injected exception or commit exception, and 2 exceptions for each stream
- if (++exceptionCount > 2 || !(e instanceof RuntimeException) ||
- !(e.getMessage().contains("test exception"))) {
- // The exception won't cause the test fail since we actually "expected" exception thrown and failed the stream.
- // So, log to stderr for debugging when the exception is not what we expected, and fail in the main thread
- e.printStackTrace(System.err);
- hasUnexpectedError = true;
- }
- exceptionCounts.put(appDir, exceptionCount);
- }
- return StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_CLIENT;
- });
-
- return streams;
- }
-
- private void waitForRunning(final List> observed) throws Exception {
- waitForCondition(
- () -> !observed.isEmpty() && observed.get(observed.size() - 1).value.equals(State.RUNNING),
- MAX_WAIT_TIME_MS,
- () -> "Client did not startup on time. Observers transitions: " + observed
- );
- }
-
- private void waitForStateTransition(final List> observed,
- final List> expected)
- throws Exception {
-
- waitForCondition(
- () -> observed.equals(expected),
- MAX_WAIT_TIME_MS,
- () -> "Client did not have the expected state transition on time. Observers transitions: " + observed
- + "Expected transitions: " + expected
- );
- }
-
- private void waitForStateTransitionContains(final List> observed,
- final List> expected)
- throws Exception {
-
- waitForCondition(
- () -> observed.containsAll(expected),
- MAX_WAIT_TIME_MS,
- () -> "Client did not have the expected state transition on time. Observers transitions: " + observed
- + "Expected transitions: " + expected
- );
- }
-
- private List> prepareData(final long fromInclusive,
- final long toExclusive,
- final Long... keys) {
- final List> data = new ArrayList<>();
-
- for (final Long k : keys) {
- for (long v = fromInclusive; v < toExclusive; ++v) {
- data.add(new KeyValue<>(k, v));
- }
- }
-
- return data;
- }
-
- private void writeInputData(final List> records) {
- final Properties config = TestUtils.producerConfig(
- CLUSTER.bootstrapServers(),
- LongSerializer.class,
- LongSerializer.class
- );
- config.setProperty(ProducerConfig.PARTITIONER_CLASS_CONFIG, KeyPartitioner.class.getName());
- IntegrationTestUtils.produceKeyValuesSynchronously(
- MULTI_PARTITION_INPUT_TOPIC,
- records,
- config,
- CLUSTER.time
- );
- }
-
- private void verifyCommitted(final List> expectedResult) throws Exception {
- final List> committedOutput = readResult(expectedResult.size(), true);
- checkResultPerKey(committedOutput, expectedResult);
- }
-
- private void verifyUncommitted(final List> expectedResult) throws Exception {
- final List> uncommittedOutput = readResult(expectedResult.size(), false);
- checkResultPerKey(uncommittedOutput, expectedResult);
- }
-
- private List> readResult(final int numberOfRecords,
- final boolean readCommitted) throws Exception {
- if (readCommitted) {
- return IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
- TestUtils.consumerConfig(
- CLUSTER.bootstrapServers(),
- CONSUMER_GROUP_ID,
- LongDeserializer.class,
- LongDeserializer.class,
- Utils.mkProperties(Collections.singletonMap(
- ConsumerConfig.ISOLATION_LEVEL_CONFIG,
- IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT))
- )
- ),
- MULTI_PARTITION_OUTPUT_TOPIC,
- numberOfRecords,
- MAX_WAIT_TIME_MS
- );
- }
-
- // read uncommitted
- return IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(
- TestUtils.consumerConfig(CLUSTER.bootstrapServers(), LongDeserializer.class, LongDeserializer.class),
- MULTI_PARTITION_OUTPUT_TOPIC,
- numberOfRecords
- );
- }
-
- private void checkResultPerKey(final List> result,
- final List> expectedResult) {
- final Set allKeys = new HashSet<>();
- addAllKeys(allKeys, result);
- addAllKeys(allKeys, expectedResult);
-
- for (final Long key : allKeys) {
- try {
- assertThat(getAllRecordPerKey(key, result), equalTo(getAllRecordPerKey(key, expectedResult)));
- } catch (final AssertionError error) {
- throw new AssertionError(
- "expected result: " + expectedResult.stream().map(KeyValue::toString).collect(Collectors.joining(", ")) +
- "\nreceived records: " + result.stream().map(KeyValue::toString).collect(Collectors.joining(", ")),
- error
- );
- }
- }
- }
-
- private void addAllKeys(final Set allKeys, final List> records) {
- for (final KeyValue record : records) {
- allKeys.add(record.key);
- }
- }
-
- private List> getAllRecordPerKey(final Long key, final List> records) {
- final List> recordsPerKey = new ArrayList<>(records.size());
-
- for (final KeyValue record : records) {
- if (record.key.equals(key)) {
- recordsPerKey.add(record);
- }
- }
-
- return recordsPerKey;
- }
-
- private List> computeExpectedResult(final List> input,
- final Map currentState) {
- final List> expectedResult = new ArrayList<>(input.size());
-
- for (final KeyValue record : input) {
- final long sum = currentState.getOrDefault(record.key, 0L);
- currentState.put(record.key, sum + record.value);
- expectedResult.add(new KeyValue<>(record.key, sum + record.value));
- }
-
- return expectedResult;
- }
-
- private Set keysFromInstance(final KafkaStreams streams) throws Exception {
- final Set keys = new HashSet<>();
- waitForCondition(
- () -> {
- final ReadOnlyKeyValueStore store = streams.store(
- StoreQueryParameters.fromNameAndType(storeName, QueryableStoreTypes.keyValueStore())
- );
-
- keys.clear();
- try (final KeyValueIterator it = store.all()) {
- while (it.hasNext()) {
- final KeyValue row = it.next();
- keys.add(row.key);
- }
- }
-
- return true;
- },
- MAX_WAIT_TIME_MS,
- "Could not get keys from store: " + storeName
- );
-
- return keys;
- }
-
- // must be public to allow KafkaProducer to instantiate it
- public static class KeyPartitioner implements Partitioner {
- private final static LongDeserializer LONG_DESERIALIZER = new LongDeserializer();
-
- @Override
- public int partition(final String topic,
- final Object key,
- final byte[] keyBytes,
- final Object value,
- final byte[] valueBytes,
- final Cluster cluster) {
- return LONG_DESERIALIZER.deserialize(topic, keyBytes).intValue() % NUM_TOPIC_PARTITIONS;
- }
-
- @Override
- public void close() {}
-
- @Override
- public void configure(final Map configs) {}
- }
-
- private class TestKafkaClientSupplier extends DefaultKafkaClientSupplier {
- @Override
- public Producer getProducer(final Map config) {
- return new ErrorInjector(config);
- }
- }
-
- private class ErrorInjector extends KafkaProducer {
- private final AtomicBoolean crash;
-
- public ErrorInjector(final Map configs) {
- super(configs, new ByteArraySerializer(), new ByteArraySerializer());
- final String clientId = configs.get(ProducerConfig.CLIENT_ID_CONFIG).toString();
- if (clientId.contains(APP_DIR_1)) {
- crash = commitErrorInjectedClient1;
- } else {
- crash = commitErrorInjectedClient2;
- }
- }
-
- @Override
- public void commitTransaction() {
- super.flush(); // we flush to ensure that the offsets are written
- if (!crash.compareAndSet(true, false)) {
- super.commitTransaction();
- } else {
- throw new RuntimeException("Injected producer commit test exception.");
- }
- }
- }
-}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableEOSIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableEOSIntegrationTest.java
index 3ac94ad96834..00e70cbfac7a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableEOSIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/GlobalKTableEOSIntegrationTest.java
@@ -56,12 +56,9 @@
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.rules.Timeout;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -71,7 +68,6 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-@RunWith(Parameterized.class)
@Category({IntegrationTest.class})
public class GlobalKTableEOSIntegrationTest {
@Rule
@@ -98,18 +94,6 @@ public static void closeCluster() {
CLUSTER.stop();
}
- @SuppressWarnings("deprecation")
- @Parameterized.Parameters(name = "{0}")
- public static Collection data() {
- return Arrays.asList(new String[][] {
- {StreamsConfig.EXACTLY_ONCE},
- {StreamsConfig.EXACTLY_ONCE_V2}
- });
- }
-
- @Parameterized.Parameter
- public String eosConfig;
-
private final MockTime mockTime = CLUSTER.time;
private final KeyValueMapper keyMapper = (key, value) -> value;
private final ValueJoiner joiner = (value1, value2) -> value1 + "+" + value2;
@@ -138,7 +122,7 @@ public void before() throws Exception {
streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0L);
streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100L);
- streamsConfiguration.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, eosConfig);
+ streamsConfiguration.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2);
streamsConfiguration.put(StreamsConfig.TASK_TIMEOUT_MS_CONFIG, 1L);
streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
streamsConfiguration.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 1000);
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KTableSourceTopicRestartIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KTableSourceTopicRestartIntegrationTest.java
index be583f17a7e2..f6ab148ff8db 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/KTableSourceTopicRestartIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/KTableSourceTopicRestartIntegrationTest.java
@@ -142,20 +142,9 @@ public void shouldRestoreAndProgressWhenTopicWrittenToDuringRestorationWithEosDi
}
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldRestoreAndProgressWhenTopicWrittenToDuringRestorationWithEosAlphaEnabled() throws Exception {
- STREAMS_CONFIG.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- shouldRestoreAndProgressWhenTopicWrittenToDuringRestorationWithEosEnabled();
- }
-
@Test
public void shouldRestoreAndProgressWhenTopicWrittenToDuringRestorationWithEosV2Enabled() throws Exception {
STREAMS_CONFIG.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2);
- shouldRestoreAndProgressWhenTopicWrittenToDuringRestorationWithEosEnabled();
- }
-
- private void shouldRestoreAndProgressWhenTopicWrittenToDuringRestorationWithEosEnabled() throws Exception {
try {
streams = new KafkaStreams(streamsBuilder.build(), STREAMS_CONFIG);
streams.start();
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java
index 7fe905ae7d4f..0eeb0b9d07cd 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/ResetPartitionTimeIntegrationTest.java
@@ -94,12 +94,10 @@ public static void closeCluster() {
private static final int DEFAULT_TIMEOUT = 100;
private static long lastRecordedTimestamp = -2L;
- @SuppressWarnings("deprecation")
@Parameterized.Parameters(name = "{0}")
public static Collection data() {
return Arrays.asList(new String[][] {
{StreamsConfig.AT_LEAST_ONCE},
- {StreamsConfig.EXACTLY_ONCE},
{StreamsConfig.EXACTLY_ONCE_V2}
});
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
index 17610c8450dd..61cfa2774958 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/RocksDBMetricsIntegrationTest.java
@@ -137,12 +137,10 @@ public static void closeCluster() {
private static final String ESTIMATED_MEMORY_OF_TABLE_READERS = "estimate-table-readers-mem";
private static final String NUMBER_OF_BACKGROUND_ERRORS = "background-errors";
- @SuppressWarnings("deprecation")
@Parameters(name = "{0}")
public static Collection data() {
return Arrays.asList(new Object[][] {
{StreamsConfig.AT_LEAST_ONCE},
- {StreamsConfig.EXACTLY_ONCE},
{StreamsConfig.EXACTLY_ONCE_V2}
});
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java
index ed68a1f33ae0..a620c995f57d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java
@@ -49,13 +49,10 @@
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.rules.Timeout;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import java.io.File;
import java.io.IOException;
import java.time.Duration;
-import java.util.Collection;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
@@ -72,7 +69,6 @@
* An integration test to verify the conversion of a dirty-closed EOS
* task towards a standby task is safe across restarts of the application.
*/
-@RunWith(Parameterized.class)
@Category(IntegrationTest.class)
public class StandbyTaskEOSIntegrationTest {
@Rule
@@ -81,18 +77,6 @@ public class StandbyTaskEOSIntegrationTest {
private final static int KEY_0 = 0;
private final static int KEY_1 = 1;
- @SuppressWarnings("deprecation")
- @Parameterized.Parameters(name = "{0}")
- public static Collection data() {
- return asList(new String[][] {
- {StreamsConfig.EXACTLY_ONCE},
- {StreamsConfig.EXACTLY_ONCE_V2}
- });
- }
-
- @Parameterized.Parameter
- public String eosConfig;
-
private final AtomicBoolean skipRecord = new AtomicBoolean(false);
private String appId;
@@ -406,7 +390,7 @@ private Properties props(final String stateDirPath) {
streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, stateDirPath);
streamsConfiguration.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
- streamsConfiguration.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, eosConfig);
+ streamsConfiguration.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2);
streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000L);
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java
index 1dc6e6a60729..58740fd99592 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/SuppressionDurabilityIntegrationTest.java
@@ -114,12 +114,10 @@ public static void closeCluster() {
private static final LongDeserializer LONG_DESERIALIZER = new LongDeserializer();
private static final long COMMIT_INTERVAL = 100L;
- @SuppressWarnings("deprecation")
@Parameterized.Parameters(name = "{0}")
public static Collection data() {
return Arrays.asList(new String[][] {
{StreamsConfig.AT_LEAST_ONCE},
- {StreamsConfig.EXACTLY_ONCE},
{StreamsConfig.EXACTLY_ONCE_V2}
});
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java
index 538360bd6313..8e424c84d705 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java
@@ -58,7 +58,6 @@
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.closeTo;
-import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.assertThrows;
import static java.util.Collections.emptySet;
@@ -109,16 +108,6 @@ public void shouldCloseThreadProducerIfEosDisabled() {
assertThat(mockClientSupplier.producers.get(0).closed(), is(true));
}
- @Test
- public void shouldNoOpCloseTaskProducerIfEosDisabled() {
- createTasks();
-
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 0));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 1));
-
- assertThat(mockClientSupplier.producers.get(0).closed(), is(false));
- }
-
@Test
public void shouldReturnBlockedTimeWhenThreadProducer() {
final double blockedTime = 123.0;
@@ -131,18 +120,6 @@ public void shouldReturnBlockedTimeWhenThreadProducer() {
// error handling
- @Test
- public void shouldFailOnStreamsProducerPerTaskIfEosDisabled() {
- createTasks();
-
- final IllegalStateException thrown = assertThrows(
- IllegalStateException.class,
- () -> activeTaskCreator.streamsProducerForTask(null)
- );
-
- assertThat(thrown.getMessage(), is("Expected EXACTLY_ONCE to be enabled, but the processing mode was AT_LEAST_ONCE"));
- }
-
@Test
public void shouldFailOnGetThreadProducerIfEosDisabled() {
createTasks();
@@ -169,155 +146,6 @@ public void shouldThrowStreamsExceptionOnErrorCloseThreadProducerIfEosDisabled()
assertThat(thrown.getCause().getMessage(), is("KABOOM!"));
}
-
-
- // eos-alpha test
-
- // functional test
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldReturnStreamsProducerPerTaskIfEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
-
- shouldReturnStreamsProducerPerTask();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldConstructProducerMetricsWithEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
-
- shouldConstructProducerMetricsPerTask();
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldConstructClientIdWithEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- mockClientSupplier.setApplicationIdForProducer("appId");
- createTasks();
-
- final Set clientIds = activeTaskCreator.producerClientIds();
-
- assertThat(clientIds, is(mkSet("clientId-StreamThread-0-0_0-producer", "clientId-StreamThread-0-0_1-producer")));
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNoOpCloseThreadProducerIfEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- mockClientSupplier.setApplicationIdForProducer("appId");
- createTasks();
-
- activeTaskCreator.closeThreadProducerIfNeeded();
-
- assertThat(mockClientSupplier.producers.get(0).closed(), is(false));
- assertThat(mockClientSupplier.producers.get(1).closed(), is(false));
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldCloseTaskProducersIfEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- mockClientSupplier.setApplicationIdForProducer("appId");
- createTasks();
-
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 0));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 1));
- // should no-op unknown task
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 2));
-
- assertThat(mockClientSupplier.producers.get(0).closed(), is(true));
- assertThat(mockClientSupplier.producers.get(1).closed(), is(true));
-
- // should not throw because producer should be removed
- mockClientSupplier.producers.get(0).closeException = new RuntimeException("KABOOM!");
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 0));
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldReturnBlockedTimeWhenTaskProducers() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- mockClientSupplier.setApplicationIdForProducer("appId");
- createTasks();
- double total = 0.0;
- double blocked = 1.0;
- for (final MockProducer producer : mockClientSupplier.producers) {
- addMetric(producer, "flush-time-ns-total", blocked);
- total += blocked;
- blocked += 1.0;
- }
-
- assertThat(activeTaskCreator.totalProducerBlockedTime(), closeTo(total, 0.01));
- }
-
- // error handling
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldFailForUnknownTaskOnStreamsProducerPerTaskIfEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- mockClientSupplier.setApplicationIdForProducer("appId");
-
- createTasks();
-
- {
- final IllegalStateException thrown = assertThrows(
- IllegalStateException.class,
- () -> activeTaskCreator.streamsProducerForTask(null)
- );
-
- assertThat(thrown.getMessage(), is("Unknown TaskId: null"));
- }
- {
- final IllegalStateException thrown = assertThrows(
- IllegalStateException.class,
- () -> activeTaskCreator.streamsProducerForTask(new TaskId(0, 2))
- );
-
- assertThat(thrown.getMessage(), is("Unknown TaskId: 0_2"));
- }
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldFailOnGetThreadProducerIfEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- mockClientSupplier.setApplicationIdForProducer("appId");
-
- createTasks();
-
- final IllegalStateException thrown = assertThrows(
- IllegalStateException.class,
- activeTaskCreator::threadProducer
- );
-
- assertThat(thrown.getMessage(), is("Expected EXACTLY_ONCE_V2 to be enabled, but the processing mode was EXACTLY_ONCE_ALPHA"));
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldThrowStreamsExceptionOnErrorCloseTaskProducerIfEosAlphaEnabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- mockClientSupplier.setApplicationIdForProducer("appId");
- createTasks();
- mockClientSupplier.producers.get(0).closeException = new RuntimeException("KABOOM!");
-
- final StreamsException thrown = assertThrows(
- StreamsException.class,
- () -> activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 0))
- );
-
- assertThat(thrown.getMessage(), is("[0_0] task producer encounter error trying to close."));
- assertThat(thrown.getCause().getMessage(), is("KABOOM!"));
-
- // should not throw again because producer should be removed
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 0));
- }
-
-
// eos-v2 test
// functional test
@@ -365,36 +193,8 @@ public void shouldCloseThreadProducerIfEosV2Enabled() {
assertThat(mockClientSupplier.producers.get(0).closed(), is(true));
}
- @Test
- public void shouldNoOpCloseTaskProducerIfEosV2Enabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2);
- mockClientSupplier.setApplicationIdForProducer("appId");
-
- createTasks();
-
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 0));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(new TaskId(0, 1));
-
- assertThat(mockClientSupplier.producers.get(0).closed(), is(false));
- }
-
// error handling
- @Test
- public void shouldFailOnStreamsProducerPerTaskIfEosV2Enabled() {
- properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2);
- mockClientSupplier.setApplicationIdForProducer("appId");
-
- createTasks();
-
- final IllegalStateException thrown = assertThrows(
- IllegalStateException.class,
- () -> activeTaskCreator.streamsProducerForTask(null)
- );
-
- assertThat(thrown.getMessage(), is("Expected EXACTLY_ONCE to be enabled, but the processing mode was EXACTLY_ONCE_V2"));
- }
-
@Test
public void shouldThrowStreamsExceptionOnErrorCloseThreadProducerIfEosV2Enabled() {
properties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2);
@@ -411,44 +211,6 @@ public void shouldThrowStreamsExceptionOnErrorCloseThreadProducerIfEosV2Enabled(
assertThat(thrown.getCause().getMessage(), is("KABOOM!"));
}
- private void shouldReturnStreamsProducerPerTask() {
- mockClientSupplier.setApplicationIdForProducer("appId");
-
- createTasks();
-
- final StreamsProducer streamsProducer1 = activeTaskCreator.streamsProducerForTask(new TaskId(0, 0));
- final StreamsProducer streamsProducer2 = activeTaskCreator.streamsProducerForTask(new TaskId(0, 1));
-
- assertThat(streamsProducer1, not(is(streamsProducer2)));
- }
-
- private void shouldConstructProducerMetricsPerTask() {
- mockClientSupplier.setApplicationIdForProducer("appId");
-
- createTasks();
-
- final MetricName testMetricName1 = new MetricName("test_metric_1", "", "", new HashMap<>());
- final Metric testMetric1 = new KafkaMetric(
- new Object(),
- testMetricName1,
- (Measurable) (config, now) -> 0,
- null,
- new MockTime());
- mockClientSupplier.producers.get(0).setMockMetrics(testMetricName1, testMetric1);
- final MetricName testMetricName2 = new MetricName("test_metric_2", "", "", new HashMap<>());
- final Metric testMetric2 = new KafkaMetric(
- new Object(),
- testMetricName2,
- (Measurable) (config, now) -> 0,
- null,
- new MockTime());
- mockClientSupplier.producers.get(0).setMockMetrics(testMetricName2, testMetric2);
-
- final Map producerMetrics = activeTaskCreator.producerMetrics();
-
- assertThat(producerMetrics, is(mkMap(mkEntry(testMetricName1, testMetric1), mkEntry(testMetricName2, testMetric2))));
- }
-
private void shouldConstructThreadProducerMetric() {
createTasks();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
index b3fa516a3f7e..cde18e615bef 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
@@ -142,7 +142,6 @@ public void setup() {
config,
processId + "-StreamThread-1",
clientSupplier,
- null,
processId,
logContext,
Time.SYSTEM
@@ -907,7 +906,6 @@ public void abortTransaction() {
};
}
},
- taskId,
processId,
logContext,
Time.SYSTEM
@@ -941,7 +939,6 @@ public List partitionsFor(final String topic) {
}
},
null,
- null,
logContext,
Time.SYSTEM
),
@@ -976,7 +973,6 @@ public Producer getProducer(final Map config) {
return mockProducer;
}
},
- taskId,
processId,
logContext,
Time.SYSTEM
@@ -1017,7 +1013,6 @@ public synchronized Future send(final ProducerRecord partitionsFor(final String topic) {
}
},
null,
- null,
logContext,
Time.SYSTEM
);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
index ba484d210ca5..9b4f924587d9 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java
@@ -504,37 +504,6 @@ public void shouldCloseStateManagerOnTaskCreated() {
assertEquals(Task.State.CLOSED, task.state());
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldDeleteStateDirOnTaskCreatedAndEosAlphaUncleanClose() {
- stateManager.close();
- EasyMock.expectLastCall();
-
- EasyMock.expect(stateManager.baseDir()).andReturn(baseDir);
-
- EasyMock.replay(stateManager);
-
- final MetricName metricName = setupCloseTaskMetric();
-
- config = new StreamsConfig(mkProperties(mkMap(
- mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, applicationId),
- mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2171"),
- mkEntry(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE)
- )));
-
- task = createStandbyTask();
- task.suspend();
-
- task.closeDirty();
-
- final double expectedCloseTaskMetric = 1.0;
- verifyCloseTaskMetric(expectedCloseTaskMetric, streamsMetrics, metricName);
-
- EasyMock.verify(stateManager);
-
- assertEquals(Task.State.CLOSED, task.state());
- }
-
@Test
public void shouldDeleteStateDirOnTaskCreatedAndEosV2UncleanClose() {
stateManager.close();
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
index 61b8791af748..451fa6902e05 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java
@@ -582,29 +582,6 @@ public void shouldProcessRecordsAfterPrepareCommitWhenEosDisabled() {
assertFalse(task.process(time.milliseconds()));
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldNotProcessRecordsAfterPrepareCommitWhenEosAlphaEnabled() {
- task = createSingleSourceStateless(createConfig(StreamsConfig.EXACTLY_ONCE, "0"), StreamsConfig.METRICS_LATEST);
-
- assertFalse(task.process(time.milliseconds()));
-
- task.addRecords(partition1, asList(
- getConsumerRecordWithOffsetAsTimestamp(partition1, 10),
- getConsumerRecordWithOffsetAsTimestamp(partition1, 20),
- getConsumerRecordWithOffsetAsTimestamp(partition1, 30)
- ));
-
- assertTrue(task.process(time.milliseconds()));
- task.prepareCommit();
- assertFalse(task.process(time.milliseconds()));
- task.postCommit(false);
- assertTrue(task.process(time.milliseconds()));
- assertTrue(task.process(time.milliseconds()));
-
- assertFalse(task.process(time.milliseconds()));
- }
-
@Test
public void shouldNotProcessRecordsAfterPrepareCommitWhenEosV2Enabled() {
task = createSingleSourceStateless(createConfig(StreamsConfig.EXACTLY_ONCE_V2, "0"), StreamsConfig.METRICS_LATEST);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
index 9e2dcdfa71ca..d769d14221b9 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java
@@ -978,44 +978,6 @@ public void shouldInjectSharedProducerForAllTasksUsingClientSupplierOnCreateIfEo
assertSame(clientSupplier.restoreConsumer, thread.restoreConsumer());
}
- @SuppressWarnings("deprecation")
- @Test
- public void shouldInjectProducerPerTaskUsingClientSupplierOnCreateIfEosAlphaEnabled() {
- internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
-
- final Properties props = configProps(true);
- props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
- final StreamThread thread = createStreamThread(CLIENT_ID, new StreamsConfig(props), true);
-
- thread.setState(StreamThread.State.STARTING);
- thread.rebalanceListener().onPartitionsRevoked(Collections.emptyList());
-
- final Map> activeTasks = new HashMap<>();
- final List assignedPartitions = new ArrayList<>();
-
- // assign single partition
- assignedPartitions.add(t1p1);
- assignedPartitions.add(t1p2);
- activeTasks.put(task1, Collections.singleton(t1p1));
- activeTasks.put(task2, Collections.singleton(t1p2));
-
- thread.taskManager().handleAssignment(activeTasks, emptyMap());
-
- final MockConsumer mockConsumer = (MockConsumer) thread.mainConsumer();
- mockConsumer.assign(assignedPartitions);
- final Map beginOffsets = new HashMap<>();
- beginOffsets.put(t1p1, 0L);
- beginOffsets.put(t1p2, 0L);
- mockConsumer.updateBeginningOffsets(beginOffsets);
- thread.rebalanceListener().onPartitionsAssigned(new HashSet<>(assignedPartitions));
-
- thread.runOnce();
-
- assertEquals(thread.activeTasks().size(), clientSupplier.producers.size());
- assertSame(clientSupplier.consumer, thread.mainConsumer());
- assertSame(clientSupplier.restoreConsumer, thread.restoreConsumer());
- }
-
@Test
public void shouldInjectProducerPerThreadUsingClientSupplierOnCreateIfEosV2Enabled() {
internalTopologyBuilder.addSource(null, "source1", null, null, null, topic1);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsProducerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsProducerTest.java
index 9470a7b166e1..bdcfbc9a3a87 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsProducerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsProducerTest.java
@@ -40,7 +40,6 @@
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.kafka.streams.errors.TaskMigratedException;
-import org.apache.kafka.streams.processor.TaskId;
import org.apache.kafka.test.MockClientSupplier;
import org.junit.Before;
import org.junit.Test;
@@ -93,14 +92,7 @@ public class StreamsProducerTest {
mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234"))
);
- @SuppressWarnings("deprecation")
- private final StreamsConfig eosAlphaConfig = new StreamsConfig(mkMap(
- mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "appId"),
- mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234"),
- mkEntry(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE))
- );
-
- private final StreamsConfig eosBetaConfig = new StreamsConfig(mkMap(
+ private final StreamsConfig eosV2Config = new StreamsConfig(mkMap(
mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "appId"),
mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234"),
mkEntry(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2))
@@ -120,16 +112,14 @@ public Producer getProducer(final Map config) {
"threadId",
clientSupplier,
null,
- null,
logContext,
mockTime
);
- final StreamsProducer eosAlphaStreamsProducerWithMock = new StreamsProducer(
- eosAlphaConfig,
- "threadId",
+ final StreamsProducer eosV2StreamsProducerWithMock = new StreamsProducer(
+ eosV2Config,
+ "threadId-StreamThread-0",
clientSupplier,
- new TaskId(0, 0),
- null,
+ UUID.randomUUID(),
logContext,
mockTime
);
@@ -138,13 +128,9 @@ public Producer getProducer(final Map config) {
private StreamsProducer nonEosStreamsProducer;
private MockProducer nonEosMockProducer;
- private final MockClientSupplier eosAlphaMockClientSupplier = new MockClientSupplier();
- private StreamsProducer eosAlphaStreamsProducer;
- private MockProducer eosAlphaMockProducer;
-
- private final MockClientSupplier eosBetaMockClientSupplier = new MockClientSupplier();
- private StreamsProducer eosBetaStreamsProducer;
- private MockProducer eosBetaMockProducer;
+ private final MockClientSupplier eosV2MockClientSupplier = new MockClientSupplier();
+ private StreamsProducer eosV2StreamsProducer;
+ private MockProducer eosV2MockProducer;
private final ProducerRecord record =
new ProducerRecord<>(topic, 0, 0L, new byte[0], new byte[0], new RecordHeaders());
@@ -162,55 +148,35 @@ public void before() {
"threadId-StreamThread-0",
mockClientSupplier,
null,
- null,
logContext,
mockTime
);
nonEosMockProducer = mockClientSupplier.producers.get(0);
- eosAlphaMockClientSupplier.setCluster(cluster);
- eosAlphaMockClientSupplier.setApplicationIdForProducer("appId");
- eosAlphaStreamsProducer =
+ eosV2MockClientSupplier.setCluster(cluster);
+ eosV2MockClientSupplier.setApplicationIdForProducer("appId");
+ eosV2StreamsProducer =
new StreamsProducer(
- eosAlphaConfig,
+ eosV2Config,
"threadId-StreamThread-0",
- eosAlphaMockClientSupplier,
- new TaskId(0, 0),
- null,
- logContext,
- mockTime
- );
- eosAlphaStreamsProducer.initTransaction();
- eosAlphaMockProducer = eosAlphaMockClientSupplier.producers.get(0);
-
- eosBetaMockClientSupplier.setCluster(cluster);
- eosBetaMockClientSupplier.setApplicationIdForProducer("appId");
- eosBetaStreamsProducer =
- new StreamsProducer(
- eosBetaConfig,
- "threadId-StreamThread-0",
- eosBetaMockClientSupplier,
- null,
+ eosV2MockClientSupplier,
UUID.randomUUID(),
logContext,
mockTime
);
- eosBetaStreamsProducer.initTransaction();
- eosBetaMockProducer = eosBetaMockClientSupplier.producers.get(0);
+ eosV2StreamsProducer.initTransaction();
+ eosV2MockProducer = eosV2MockClientSupplier.producers.get(0);
expect(mockTime.nanoseconds()).andAnswer(Time.SYSTEM::nanoseconds).anyTimes();
replay(mockTime);
}
-
-
- // common tests (non-EOS and EOS-alpha/beta)
+ // common tests (non-EOS and EOS)
// functional tests
@Test
public void shouldCreateProducer() {
assertThat(mockClientSupplier.producers.size(), is(1));
- assertThat(eosAlphaMockClientSupplier.producers.size(), is(1));
}
@Test
@@ -269,7 +235,6 @@ public void shouldFailIfStreamsConfigIsNull() {
null,
"threadId",
mockClientSupplier,
- new TaskId(0, 0),
UUID.randomUUID(),
logContext,
mockTime)
@@ -286,7 +251,6 @@ public void shouldFailIfThreadIdIsNull() {
nonEosConfig,
null,
mockClientSupplier,
- new TaskId(0, 0),
UUID.randomUUID(),
logContext,
mockTime)
@@ -303,7 +267,6 @@ public void shouldFailIfClientSupplierIsNull() {
nonEosConfig,
"threadId",
null,
- new TaskId(0, 0),
UUID.randomUUID(),
logContext,
mockTime)
@@ -320,7 +283,6 @@ public void shouldFailIfLogContextIsNull() {
nonEosConfig,
"threadId",
mockClientSupplier,
- new TaskId(0, 0),
UUID.randomUUID(),
null,
mockTime)
@@ -339,17 +301,6 @@ public void shouldFailOnResetProducerForAtLeastOnce() {
assertThat(thrown.getMessage(), is("Expected eos-v2 to be enabled, but the processing mode was AT_LEAST_ONCE"));
}
- @Test
- public void shouldFailOnResetProducerForExactlyOnceAlpha() {
- final IllegalStateException thrown = assertThrows(
- IllegalStateException.class,
- () -> eosAlphaStreamsProducer.resetProducer()
- );
-
- assertThat(thrown.getMessage(), is("Expected eos-v2 to be enabled, but the processing mode was EXACTLY_ONCE_ALPHA"));
- }
-
-
// non-EOS tests
// functional tests
@@ -366,7 +317,6 @@ public void shouldNotSetTransactionIdIfEosDisabled() {
"threadId",
mockClientSupplier,
null,
- null,
logContext,
mockTime
);
@@ -453,45 +403,13 @@ public void shouldFailOnAbortIfEosDisabled() {
}
- // EOS tests (alpha and beta)
+ // EOS tests
// functional tests
@Test
- public void shouldEnableEosIfEosAlphaEnabled() {
- assertThat(eosAlphaStreamsProducer.eosEnabled(), is(true));
- }
-
- @Test
- public void shouldEnableEosIfEosBetaEnabled() {
- assertThat(eosBetaStreamsProducer.eosEnabled(), is(true));
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void shouldSetTransactionIdUsingTaskIdIfEosAlphaEnabled() {
- final Map mockMap = mock(Map.class);
- expect(mockMap.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "appId-0_0")).andReturn(null);
- expect(mockMap.get(ProducerConfig.TRANSACTIONAL_ID_CONFIG)).andReturn("appId-0_0");
-
- final StreamsConfig mockConfig = mock(StreamsConfig.class);
- expect(mockConfig.getProducerConfigs("threadId-0_0-producer")).andReturn(mockMap);
- expect(mockConfig.getString(StreamsConfig.APPLICATION_ID_CONFIG)).andReturn("appId");
- expect(mockConfig.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG)).andReturn(StreamsConfig.EXACTLY_ONCE);
-
- replay(mockMap, mockConfig);
-
- new StreamsProducer(
- mockConfig,
- "threadId",
- eosAlphaMockClientSupplier,
- new TaskId(0, 0),
- null,
- logContext,
- mockTime
- );
-
- verify(mockMap);
+ public void shouldEnableEosIfEosV2Enabled() {
+ assertThat(eosV2StreamsProducer.eosEnabled(), is(true));
}
@Test
@@ -512,8 +430,7 @@ public void shouldSetTransactionIdUsingProcessIdIfEosV2Enabled() {
new StreamsProducer(
mockConfig,
"threadId-StreamThread-0",
- eosAlphaMockClientSupplier,
- null,
+ eosV2MockClientSupplier,
processId,
logContext,
mockTime
@@ -523,41 +440,36 @@ public void shouldSetTransactionIdUsingProcessIdIfEosV2Enabled() {
}
@Test
- public void shouldNotHaveEosEnabledIfEosAlphaEnable() {
- assertThat(eosAlphaStreamsProducer.eosEnabled(), is(true));
- }
-
- @Test
- public void shouldHaveEosEnabledIfEosBetaEnabled() {
- assertThat(eosBetaStreamsProducer.eosEnabled(), is(true));
+ public void shouldHaveEosEnabledIfEosV2Enabled() {
+ assertThat(eosV2StreamsProducer.eosEnabled(), is(true));
}
@Test
public void shouldInitTxOnEos() {
- assertThat(eosAlphaMockProducer.transactionInitialized(), is(true));
+ assertThat(eosV2MockProducer.transactionInitialized(), is(true));
}
@Test
public void shouldBeginTxOnEosSend() {
- eosAlphaStreamsProducer.send(record, null);
- assertThat(eosAlphaMockProducer.transactionInFlight(), is(true));
+ eosV2StreamsProducer.send(record, null);
+ assertThat(eosV2MockProducer.transactionInFlight(), is(true));
}
@Test
public void shouldContinueTxnSecondEosSend() {
- eosAlphaStreamsProducer.send(record, null);
- eosAlphaStreamsProducer.send(record, null);
- assertThat(eosAlphaMockProducer.transactionInFlight(), is(true));
- assertThat(eosAlphaMockProducer.uncommittedRecords().size(), is(2));
+ eosV2StreamsProducer.send(record, null);
+ eosV2StreamsProducer.send(record, null);
+ assertThat(eosV2MockProducer.transactionInFlight(), is(true));
+ assertThat(eosV2MockProducer.uncommittedRecords().size(), is(2));
}
@Test
public void shouldForwardRecordButNotCommitOnEosSend() {
- eosAlphaStreamsProducer.send(record, null);
- assertThat(eosAlphaMockProducer.transactionInFlight(), is(true));
- assertThat(eosAlphaMockProducer.history().isEmpty(), is(true));
- assertThat(eosAlphaMockProducer.uncommittedRecords().size(), is(1));
- assertThat(eosAlphaMockProducer.uncommittedRecords().get(0), is(record));
+ eosV2StreamsProducer.send(record, null);
+ assertThat(eosV2MockProducer.transactionInFlight(), is(true));
+ assertThat(eosV2MockProducer.history().isEmpty(), is(true));
+ assertThat(eosV2MockProducer.uncommittedRecords().size(), is(1));
+ assertThat(eosV2MockProducer.uncommittedRecords().get(0), is(record));
}
@Test
@@ -569,37 +481,37 @@ public void shouldBeginTxOnEosCommit() {
expectLastCall();
replay(mockedProducer);
- eosAlphaStreamsProducerWithMock.initTransaction();
+ eosV2StreamsProducerWithMock.initTransaction();
- eosAlphaStreamsProducerWithMock.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"));
+ eosV2StreamsProducerWithMock.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"));
verify(mockedProducer);
}
@Test
public void shouldSendOffsetToTxOnEosCommit() {
- eosAlphaStreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"));
- assertThat(eosAlphaMockProducer.sentOffsets(), is(true));
+ eosV2StreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"));
+ assertThat(eosV2MockProducer.sentOffsets(), is(true));
}
@Test
public void shouldCommitTxOnEosCommit() {
- eosAlphaStreamsProducer.send(record, null);
- assertThat(eosAlphaMockProducer.transactionInFlight(), is(true));
+ eosV2StreamsProducer.send(record, null);
+ assertThat(eosV2MockProducer.transactionInFlight(), is(true));
- eosAlphaStreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"));
+ eosV2StreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"));
- assertThat(eosAlphaMockProducer.transactionInFlight(), is(false));
- assertThat(eosAlphaMockProducer.uncommittedRecords().isEmpty(), is(true));
- assertThat(eosAlphaMockProducer.uncommittedOffsets().isEmpty(), is(true));
- assertThat(eosAlphaMockProducer.history().size(), is(1));
- assertThat(eosAlphaMockProducer.history().get(0), is(record));
- assertThat(eosAlphaMockProducer.consumerGroupOffsetsHistory().size(), is(1));
- assertThat(eosAlphaMockProducer.consumerGroupOffsetsHistory().get(0).get("appId"), is(offsetsAndMetadata));
+ assertThat(eosV2MockProducer.transactionInFlight(), is(false));
+ assertThat(eosV2MockProducer.uncommittedRecords().isEmpty(), is(true));
+ assertThat(eosV2MockProducer.uncommittedOffsets().isEmpty(), is(true));
+ assertThat(eosV2MockProducer.history().size(), is(1));
+ assertThat(eosV2MockProducer.history().get(0), is(record));
+ assertThat(eosV2MockProducer.consumerGroupOffsetsHistory().size(), is(1));
+ assertThat(eosV2MockProducer.consumerGroupOffsetsHistory().get(0).get("appId"), is(offsetsAndMetadata));
}
@Test
- public void shouldCommitTxWithApplicationIdOnEosAlphaCommit() {
+ public void shouldCommitTxWithApplicationIdOnEosV2Commit() {
mockedProducer.initTransactions();
expectLastCall();
mockedProducer.beginTransaction();
@@ -611,17 +523,17 @@ public void shouldCommitTxWithApplicationIdOnEosAlphaCommit() {
expectLastCall();
replay(mockedProducer);
- eosAlphaStreamsProducerWithMock.initTransaction();
+ eosV2StreamsProducerWithMock.initTransaction();
// call `send()` to start a transaction
- eosAlphaStreamsProducerWithMock.send(record, null);
+ eosV2StreamsProducerWithMock.send(record, null);
- eosAlphaStreamsProducerWithMock.commitTransaction(null, new ConsumerGroupMetadata("appId"));
+ eosV2StreamsProducerWithMock.commitTransaction(null, new ConsumerGroupMetadata("appId"));
verify(mockedProducer);
}
@Test
- public void shouldCommitTxWithConsumerGroupMetadataOnEosBetaCommit() {
+ public void shouldCommitTxWithConsumerGroupMetadataOnEosV2Commit() {
mockedProducer.initTransactions();
expectLastCall();
mockedProducer.beginTransaction();
@@ -634,10 +546,9 @@ public void shouldCommitTxWithConsumerGroupMetadataOnEosBetaCommit() {
replay(mockedProducer);
final StreamsProducer streamsProducer = new StreamsProducer(
- eosBetaConfig,
+ eosV2Config,
"threadId-StreamThread-0",
clientSupplier,
- null,
UUID.randomUUID(),
logContext,
mockTime
@@ -654,18 +565,18 @@ public void shouldCommitTxWithConsumerGroupMetadataOnEosBetaCommit() {
@Test
public void shouldAbortTxOnEosAbort() {
// call `send()` to start a transaction
- eosAlphaStreamsProducer.send(record, null);
- assertThat(eosAlphaMockProducer.transactionInFlight(), is(true));
- assertThat(eosAlphaMockProducer.uncommittedRecords().size(), is(1));
- assertThat(eosAlphaMockProducer.uncommittedRecords().get(0), is(record));
+ eosV2StreamsProducer.send(record, null);
+ assertThat(eosV2MockProducer.transactionInFlight(), is(true));
+ assertThat(eosV2MockProducer.uncommittedRecords().size(), is(1));
+ assertThat(eosV2MockProducer.uncommittedRecords().get(0), is(record));
- eosAlphaStreamsProducer.abortTransaction();
+ eosV2StreamsProducer.abortTransaction();
- assertThat(eosAlphaMockProducer.transactionInFlight(), is(false));
- assertThat(eosAlphaMockProducer.uncommittedRecords().isEmpty(), is(true));
- assertThat(eosAlphaMockProducer.uncommittedOffsets().isEmpty(), is(true));
- assertThat(eosAlphaMockProducer.history().isEmpty(), is(true));
- assertThat(eosAlphaMockProducer.consumerGroupOffsetsHistory().isEmpty(), is(true));
+ assertThat(eosV2MockProducer.transactionInFlight(), is(false));
+ assertThat(eosV2MockProducer.uncommittedRecords().isEmpty(), is(true));
+ assertThat(eosV2MockProducer.uncommittedOffsets().isEmpty(), is(true));
+ assertThat(eosV2MockProducer.history().isEmpty(), is(true));
+ assertThat(eosV2MockProducer.consumerGroupOffsetsHistory().isEmpty(), is(true));
}
@Test
@@ -674,9 +585,9 @@ public void shouldSkipAbortTxOnEosAbortIfNotTxInFlight() {
expectLastCall();
replay(mockedProducer);
- eosAlphaStreamsProducerWithMock.initTransaction();
+ eosV2StreamsProducerWithMock.initTransaction();
- eosAlphaStreamsProducerWithMock.abortTransaction();
+ eosV2StreamsProducerWithMock.abortTransaction();
verify(mockedProducer);
}
@@ -684,32 +595,14 @@ public void shouldSkipAbortTxOnEosAbortIfNotTxInFlight() {
// error handling tests
@Test
- public void shouldFailIfTaskIdIsNullForEosAlpha() {
+ public void shouldFailIfProcessIdNullForEosV2() {
final NullPointerException thrown = assertThrows(
NullPointerException.class,
() -> new StreamsProducer(
- eosAlphaConfig,
+ eosV2Config,
"threadId",
mockClientSupplier,
null,
- UUID.randomUUID(),
- logContext,
- mockTime)
- );
-
- assertThat(thrown.getMessage(), is("taskId cannot be null for exactly-once alpha"));
- }
-
- @Test
- public void shouldFailIfProcessIdNullForEosBeta() {
- final NullPointerException thrown = assertThrows(
- NullPointerException.class,
- () -> new StreamsProducer(
- eosBetaConfig,
- "threadId",
- mockClientSupplier,
- new TaskId(0, 0),
- null,
logContext,
mockTime)
);
@@ -729,11 +622,10 @@ public Producer getProducer(final Map config) {
};
final StreamsProducer streamsProducer = new StreamsProducer(
- eosAlphaConfig,
- "threadId",
+ eosV2Config,
+ "threadId-StreamThread-0",
clientSupplier,
- new TaskId(0, 0),
- null,
+ UUID.randomUUID(),
logContext,
mockTime
);
@@ -747,34 +639,12 @@ public Producer getProducer(final Map config) {
}
@Test
- public void shouldFailOnMaybeBeginTransactionIfTransactionsNotInitializedForExactlyOnceAlpha() {
- final StreamsProducer streamsProducer =
- new StreamsProducer(
- eosAlphaConfig,
- "threadId",
- eosAlphaMockClientSupplier,
- new TaskId(0, 0),
- null,
- logContext,
- mockTime
- );
-
- final IllegalStateException thrown = assertThrows(
- IllegalStateException.class,
- () -> streamsProducer.send(record, null)
- );
-
- assertThat(thrown.getMessage(), is("MockProducer hasn't been initialized for transactions."));
- }
-
- @Test
- public void shouldFailOnMaybeBeginTransactionIfTransactionsNotInitializedForExactlyOnceBeta() {
+ public void shouldFailOnMaybeBeginTransactionIfTransactionsNotInitializedForExactlyOnceV2() {
final StreamsProducer streamsProducer =
new StreamsProducer(
- eosBetaConfig,
+ eosV2Config,
"threadId-StreamThread-0",
- eosBetaMockClientSupplier,
- null,
+ eosV2MockClientSupplier,
UUID.randomUUID(),
logContext,
mockTime
@@ -800,11 +670,10 @@ public Producer getProducer(final Map config) {
};
final StreamsProducer streamsProducer = new StreamsProducer(
- eosAlphaConfig,
- "threadId",
+ eosV2Config,
+ "threadId-StreamThread-0",
clientSupplier,
- new TaskId(0, 0),
- null,
+ UUID.randomUUID(),
logContext,
mockTime
);
@@ -830,11 +699,10 @@ public Producer getProducer(final Map config) {
};
final StreamsProducer streamsProducer = new StreamsProducer(
- eosAlphaConfig,
- "threadId",
+ eosV2Config,
+ "threadId-StreamThread-0",
clientSupplier,
- new TaskId(0, 0),
- null,
+ UUID.randomUUID(),
logContext,
mockTime
);
@@ -849,11 +717,11 @@ public Producer getProducer(final Map config) {
@Test
public void shouldThrowTaskMigrateExceptionOnEosBeginTxnFenced() {
- eosAlphaMockProducer.fenceProducer();
+ eosV2MockProducer.fenceProducer();
final TaskMigratedException thrown = assertThrows(
TaskMigratedException.class,
- () -> eosAlphaStreamsProducer.send(null, null)
+ () -> eosV2StreamsProducer.send(null, null)
);
assertThat(
@@ -865,14 +733,14 @@ public void shouldThrowTaskMigrateExceptionOnEosBeginTxnFenced() {
@Test
public void shouldThrowTaskMigrateExceptionOnEosBeginTxnError() {
- eosAlphaMockProducer.beginTransactionException = new KafkaException("KABOOM!");
+ eosV2MockProducer.beginTransactionException = new KafkaException("KABOOM!");
// calling `send()` implicitly starts a new transaction
final StreamsException thrown = assertThrows(
StreamsException.class,
- () -> eosAlphaStreamsProducer.send(null, null));
+ () -> eosV2StreamsProducer.send(null, null));
- assertThat(thrown.getCause(), is(eosAlphaMockProducer.beginTransactionException));
+ assertThat(thrown.getCause(), is(eosV2MockProducer.beginTransactionException));
assertThat(
thrown.getMessage(),
is("Error encountered trying to begin a new transaction [test]")
@@ -881,12 +749,12 @@ public void shouldThrowTaskMigrateExceptionOnEosBeginTxnError() {
@Test
public void shouldFailOnEosBeginTxnFatal() {
- eosAlphaMockProducer.beginTransactionException = new RuntimeException("KABOOM!");
+ eosV2MockProducer.beginTransactionException = new RuntimeException("KABOOM!");
// calling `send()` implicitly starts a new transaction
final RuntimeException thrown = assertThrows(
RuntimeException.class,
- () -> eosAlphaStreamsProducer.send(null, null));
+ () -> eosV2StreamsProducer.send(null, null));
assertThat(thrown.getMessage(), is("KABOOM!"));
}
@@ -904,11 +772,11 @@ public void shouldThrowTaskMigratedExceptionOnEosSendInvalidEpoch() {
private void testThrowTaskMigratedExceptionOnEosSend(final RuntimeException exception) {
// we need to mimic that `send()` always wraps error in a KafkaException
// cannot use `eosMockProducer.fenceProducer()` because this would already trigger in `beginTransaction()`
- eosAlphaMockProducer.sendException = new KafkaException(exception);
+ eosV2MockProducer.sendException = new KafkaException(exception);
final TaskMigratedException thrown = assertThrows(
TaskMigratedException.class,
- () -> eosAlphaStreamsProducer.send(record, null)
+ () -> eosV2StreamsProducer.send(record, null)
);
assertThat(thrown.getCause(), is(exception));
@@ -923,11 +791,11 @@ private void testThrowTaskMigratedExceptionOnEosSend(final RuntimeException exce
public void shouldThrowTaskMigratedExceptionOnEosSendUnknownPid() {
final UnknownProducerIdException exception = new UnknownProducerIdException("KABOOM!");
// we need to mimic that `send()` always wraps error in a KafkaException
- eosAlphaMockProducer.sendException = new KafkaException(exception);
+ eosV2MockProducer.sendException = new KafkaException(exception);
final TaskMigratedException thrown = assertThrows(
TaskMigratedException.class,
- () -> eosAlphaStreamsProducer.send(record, null)
+ () -> eosV2StreamsProducer.send(record, null)
);
assertThat(thrown.getCause(), is(exception));
@@ -952,16 +820,16 @@ public void shouldThrowTaskMigrateExceptionOnEosSendOffsetInvalidEpoch() {
private void testThrowTaskMigrateExceptionOnEosSendOffset(final RuntimeException exception) {
// cannot use `eosMockProducer.fenceProducer()` because this would already trigger in `beginTransaction()`
- eosAlphaMockProducer.sendOffsetsToTransactionException = exception;
+ eosV2MockProducer.sendOffsetsToTransactionException = exception;
final TaskMigratedException thrown = assertThrows(
TaskMigratedException.class,
// we pass in `null` to verify that `sendOffsetsToTransaction()` fails instead of `commitTransaction()`
// `sendOffsetsToTransaction()` would throw an NPE on `null` offsets
- () -> eosAlphaStreamsProducer.commitTransaction(null, new ConsumerGroupMetadata("appId"))
+ () -> eosV2StreamsProducer.commitTransaction(null, new ConsumerGroupMetadata("appId"))
);
- assertThat(thrown.getCause(), is(eosAlphaMockProducer.sendOffsetsToTransactionException));
+ assertThat(thrown.getCause(), is(eosV2MockProducer.sendOffsetsToTransactionException));
assertThat(
thrown.getMessage(),
is("Producer got fenced trying to commit a transaction [test];" +
@@ -971,16 +839,16 @@ private void testThrowTaskMigrateExceptionOnEosSendOffset(final RuntimeException
@Test
public void shouldThrowStreamsExceptionOnEosSendOffsetError() {
- eosAlphaMockProducer.sendOffsetsToTransactionException = new KafkaException("KABOOM!");
+ eosV2MockProducer.sendOffsetsToTransactionException = new KafkaException("KABOOM!");
final StreamsException thrown = assertThrows(
StreamsException.class,
// we pass in `null` to verify that `sendOffsetsToTransaction()` fails instead of `commitTransaction()`
// `sendOffsetsToTransaction()` would throw an NPE on `null` offsets
- () -> eosAlphaStreamsProducer.commitTransaction(null, new ConsumerGroupMetadata("appId"))
+ () -> eosV2StreamsProducer.commitTransaction(null, new ConsumerGroupMetadata("appId"))
);
- assertThat(thrown.getCause(), is(eosAlphaMockProducer.sendOffsetsToTransactionException));
+ assertThat(thrown.getCause(), is(eosV2MockProducer.sendOffsetsToTransactionException));
assertThat(
thrown.getMessage(),
is("Error encountered trying to commit a transaction [test]")
@@ -989,13 +857,13 @@ public void shouldThrowStreamsExceptionOnEosSendOffsetError() {
@Test
public void shouldFailOnEosSendOffsetFatal() {
- eosAlphaMockProducer.sendOffsetsToTransactionException = new RuntimeException("KABOOM!");
+ eosV2MockProducer.sendOffsetsToTransactionException = new RuntimeException("KABOOM!");
final RuntimeException thrown = assertThrows(
RuntimeException.class,
// we pass in `null` to verify that `sendOffsetsToTransaction()` fails instead of `commitTransaction()`
// `sendOffsetsToTransaction()` would throw an NPE on `null` offsets
- () -> eosAlphaStreamsProducer.commitTransaction(null, new ConsumerGroupMetadata("appId"))
+ () -> eosV2StreamsProducer.commitTransaction(null, new ConsumerGroupMetadata("appId"))
);
assertThat(thrown.getMessage(), is("KABOOM!"));
@@ -1013,15 +881,15 @@ public void shouldThrowTaskMigratedExceptionOnEosCommitWithInvalidEpoch() {
private void testThrowTaskMigratedExceptionOnEos(final RuntimeException exception) {
// cannot use `eosMockProducer.fenceProducer()` because this would already trigger in `beginTransaction()`
- eosAlphaMockProducer.commitTransactionException = exception;
+ eosV2MockProducer.commitTransactionException = exception;
final TaskMigratedException thrown = assertThrows(
TaskMigratedException.class,
- () -> eosAlphaStreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"))
+ () -> eosV2StreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"))
);
- assertThat(eosAlphaMockProducer.sentOffsets(), is(true));
- assertThat(thrown.getCause(), is(eosAlphaMockProducer.commitTransactionException));
+ assertThat(eosV2MockProducer.sentOffsets(), is(true));
+ assertThat(thrown.getCause(), is(eosV2MockProducer.commitTransactionException));
assertThat(
thrown.getMessage(),
is("Producer got fenced trying to commit a transaction [test];" +
@@ -1031,15 +899,15 @@ private void testThrowTaskMigratedExceptionOnEos(final RuntimeException exceptio
@Test
public void shouldThrowStreamsExceptionOnEosCommitTxError() {
- eosAlphaMockProducer.commitTransactionException = new KafkaException("KABOOM!");
+ eosV2MockProducer.commitTransactionException = new KafkaException("KABOOM!");
final StreamsException thrown = assertThrows(
StreamsException.class,
- () -> eosAlphaStreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"))
+ () -> eosV2StreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"))
);
- assertThat(eosAlphaMockProducer.sentOffsets(), is(true));
- assertThat(thrown.getCause(), is(eosAlphaMockProducer.commitTransactionException));
+ assertThat(eosV2MockProducer.sentOffsets(), is(true));
+ assertThat(thrown.getCause(), is(eosV2MockProducer.commitTransactionException));
assertThat(
thrown.getMessage(),
is("Error encountered trying to commit a transaction [test]")
@@ -1048,14 +916,14 @@ public void shouldThrowStreamsExceptionOnEosCommitTxError() {
@Test
public void shouldFailOnEosCommitTxFatal() {
- eosAlphaMockProducer.commitTransactionException = new RuntimeException("KABOOM!");
+ eosV2MockProducer.commitTransactionException = new RuntimeException("KABOOM!");
final RuntimeException thrown = assertThrows(
RuntimeException.class,
- () -> eosAlphaStreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"))
+ () -> eosV2StreamsProducer.commitTransaction(offsetsAndMetadata, new ConsumerGroupMetadata("appId"))
);
- assertThat(eosAlphaMockProducer.sentOffsets(), is(true));
+ assertThat(eosV2MockProducer.sentOffsets(), is(true));
assertThat(thrown.getMessage(), is("KABOOM!"));
}
@@ -1077,24 +945,24 @@ private void testSwallowExceptionOnEosAbortTx(final RuntimeException exception)
expectLastCall().andThrow(exception);
replay(mockedProducer);
- eosAlphaStreamsProducerWithMock.initTransaction();
+ eosV2StreamsProducerWithMock.initTransaction();
// call `send()` to start a transaction
- eosAlphaStreamsProducerWithMock.send(record, null);
+ eosV2StreamsProducerWithMock.send(record, null);
- eosAlphaStreamsProducerWithMock.abortTransaction();
+ eosV2StreamsProducerWithMock.abortTransaction();
verify(mockedProducer);
}
@Test
public void shouldThrowStreamsExceptionOnEosAbortTxError() {
- eosAlphaMockProducer.abortTransactionException = new KafkaException("KABOOM!");
+ eosV2MockProducer.abortTransactionException = new KafkaException("KABOOM!");
// call `send()` to start a transaction
- eosAlphaStreamsProducer.send(record, null);
+ eosV2StreamsProducer.send(record, null);
- final StreamsException thrown = assertThrows(StreamsException.class, eosAlphaStreamsProducer::abortTransaction);
+ final StreamsException thrown = assertThrows(StreamsException.class, eosV2StreamsProducer::abortTransaction);
- assertThat(thrown.getCause(), is(eosAlphaMockProducer.abortTransactionException));
+ assertThat(thrown.getCause(), is(eosV2MockProducer.abortTransactionException));
assertThat(
thrown.getMessage(),
is("Error encounter trying to abort a transaction [test]")
@@ -1103,42 +971,41 @@ public void shouldThrowStreamsExceptionOnEosAbortTxError() {
@Test
public void shouldFailOnEosAbortTxFatal() {
- eosAlphaMockProducer.abortTransactionException = new RuntimeException("KABOOM!");
+ eosV2MockProducer.abortTransactionException = new RuntimeException("KABOOM!");
// call `send()` to start a transaction
- eosAlphaStreamsProducer.send(record, null);
+ eosV2StreamsProducer.send(record, null);
- final RuntimeException thrown = assertThrows(RuntimeException.class, eosAlphaStreamsProducer::abortTransaction);
+ final RuntimeException thrown = assertThrows(RuntimeException.class, eosV2StreamsProducer::abortTransaction);
assertThat(thrown.getMessage(), is("KABOOM!"));
}
- // EOS beta test
+ // EOS test
// functional tests
@Test
public void shouldCloseExistingProducerOnResetProducer() {
- eosBetaStreamsProducer.resetProducer();
+ eosV2StreamsProducer.resetProducer();
- assertTrue(eosBetaMockProducer.closed());
+ assertTrue(eosV2MockProducer.closed());
}
@Test
public void shouldSetNewProducerOnResetProducer() {
- eosBetaStreamsProducer.resetProducer();
+ eosV2StreamsProducer.resetProducer();
- assertThat(eosBetaMockClientSupplier.producers.size(), is(2));
- assertThat(eosBetaStreamsProducer.kafkaProducer(), is(eosBetaMockClientSupplier.producers.get(1)));
+ assertThat(eosV2MockClientSupplier.producers.size(), is(2));
+ assertThat(eosV2StreamsProducer.kafkaProducer(), is(eosV2MockClientSupplier.producers.get(1)));
}
@Test
public void shouldResetTransactionInitializedOnResetProducer() {
final StreamsProducer streamsProducer = new StreamsProducer(
- eosBetaConfig,
+ eosV2Config,
"threadId-StreamThread-0",
clientSupplier,
- null,
UUID.randomUUID(),
logContext,
mockTime
@@ -1181,7 +1048,7 @@ public void shouldComputeTotalBlockedTime() {
@Test
public void shouldComputeTotalBlockedTimeAfterReset() {
setProducerMetrics(
- eosBetaMockProducer,
+ eosV2MockProducer,
BUFFER_POOL_WAIT_TIME,
FLUSH_TME,
TXN_INIT_TIME,
@@ -1194,15 +1061,15 @@ public void shouldComputeTotalBlockedTimeAfterReset() {
final double expectedTotalBlocked = BUFFER_POOL_WAIT_TIME + FLUSH_TME + TXN_INIT_TIME +
TXN_BEGIN_TIME + TXN_SEND_OFFSETS_TIME + TXN_COMMIT_TIME + TXN_ABORT_TIME +
METADATA_WAIT_TIME;
- assertThat(eosBetaStreamsProducer.totalBlockedTime(), equalTo(expectedTotalBlocked));
+ assertThat(eosV2StreamsProducer.totalBlockedTime(), equalTo(expectedTotalBlocked));
reset(mockTime);
final long closeStart = 1L;
final long clodeDelay = 1L;
expect(mockTime.nanoseconds()).andReturn(closeStart).andReturn(closeStart + clodeDelay);
replay(mockTime);
- eosBetaStreamsProducer.resetProducer();
+ eosV2StreamsProducer.resetProducer();
setProducerMetrics(
- eosBetaMockClientSupplier.producers.get(1),
+ eosV2MockClientSupplier.producers.get(1),
BUFFER_POOL_WAIT_TIME,
FLUSH_TME,
TXN_INIT_TIME,
@@ -1214,7 +1081,7 @@ public void shouldComputeTotalBlockedTimeAfterReset() {
);
assertThat(
- eosBetaStreamsProducer.totalBlockedTime(),
+ eosV2StreamsProducer.totalBlockedTime(),
closeTo(2 * expectedTotalBlocked + clodeDelay, 0.01)
);
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
index 82683ec44888..598e01bb6203 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java
@@ -123,7 +123,6 @@
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.argThat;
-import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.when;
@@ -524,7 +523,6 @@ public void shouldAssignActiveTaskInTasksRegistryToBeRecycledWithStateUpdaterEna
final TaskManager taskManager = setUpTaskManager(ProcessingMode.AT_LEAST_ONCE, tasks, true);
expect(standbyTaskCreator.createStandbyTaskFromActive(activeTaskToRecycle, activeTaskToRecycle.inputPartitions()))
.andReturn(recycledStandbyTask);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(activeTaskToRecycle.id());
expect(activeTaskCreator.createTasks(consumer, Collections.emptyMap())).andReturn(emptySet());
expect(standbyTaskCreator.createTasks(Collections.emptyMap())).andReturn(emptySet());
replay(activeTaskCreator, standbyTaskCreator);
@@ -569,7 +567,6 @@ public void shouldAssignActiveTaskInTasksRegistryToBeClosedCleanlyWithStateUpdat
final TasksRegistry tasks = Mockito.mock(TasksRegistry.class);
final TaskManager taskManager = setUpTaskManager(ProcessingMode.AT_LEAST_ONCE, tasks, true);
when(tasks.allTasks()).thenReturn(mkSet(activeTaskToClose));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(activeTaskToClose.id());
expect(activeTaskCreator.createTasks(consumer, Collections.emptyMap())).andReturn(emptySet());
expect(standbyTaskCreator.createTasks(Collections.emptyMap())).andReturn(emptySet());
replay(activeTaskCreator, standbyTaskCreator);
@@ -701,7 +698,6 @@ public void shouldAssignMultipleTasksInTasksRegistryWithStateUpdaterEnabled() {
final TasksRegistry tasks = Mockito.mock(TasksRegistry.class);
final TaskManager taskManager = setUpTaskManager(ProcessingMode.AT_LEAST_ONCE, tasks, true);
when(tasks.allTasks()).thenReturn(mkSet(activeTaskToClose));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(activeTaskToClose.id());
expect(activeTaskCreator.createTasks(
consumer,
mkMap(mkEntry(activeTaskToCreate.id(), activeTaskToCreate.inputPartitions()))
@@ -757,8 +753,6 @@ public void shouldRecycleTasksRemovedFromStateUpdater() {
taskManager = setUpTaskManager(StreamsConfigUtils.ProcessingMode.AT_LEAST_ONCE, tasks, true);
expect(activeTaskCreator.createActiveTaskFromStandby(eq(task01), eq(taskId01Partitions), eq(consumer)))
.andStubReturn(task01Converted);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(anyObject());
- expectLastCall().once();
expect(standbyTaskCreator.createStandbyTaskFromActive(eq(task00), eq(taskId00Partitions)))
.andStubReturn(task00Converted);
replay(activeTaskCreator, standbyTaskCreator);
@@ -788,8 +782,6 @@ public void shouldCloseTasksRemovedFromStateUpdater() {
when(tasks.removePendingTaskToCloseClean(task00.id())).thenReturn(true);
when(tasks.removePendingTaskToCloseClean(task01.id())).thenReturn(true);
taskManager = setUpTaskManager(StreamsConfigUtils.ProcessingMode.AT_LEAST_ONCE, tasks, true);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(anyObject());
- expectLastCall().once();
replay(activeTaskCreator);
taskManager.checkStateUpdater(time.milliseconds(), noOpResetter);
@@ -869,8 +861,6 @@ public void shouldHandleMultipleRemovedTasksFromStateUpdater() {
when(stateUpdater.restoresActiveTasks()).thenReturn(true);
expect(activeTaskCreator.createActiveTaskFromStandby(eq(taskToRecycle1), eq(taskId01Partitions), eq(consumer)))
.andStubReturn(convertedTask1);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(anyObject());
- expectLastCall().times(2);
expect(standbyTaskCreator.createStandbyTaskFromActive(eq(taskToRecycle0), eq(taskId00Partitions)))
.andStubReturn(convertedTask0);
expect(consumer.assignment()).andReturn(emptySet()).anyTimes();
@@ -1061,7 +1051,6 @@ public void shouldRecycleRestoredTask() {
final TaskManager taskManager = setUpRecycleRestoredTask(statefulTask);
expect(standbyTaskCreator.createStandbyTaskFromActive(statefulTask, statefulTask.inputPartitions()))
.andStubReturn(standbyTask);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(statefulTask.id());
replay(activeTaskCreator, standbyTaskCreator);
taskManager.checkStateUpdater(time.milliseconds(), noOpResetter);
@@ -1132,7 +1121,6 @@ public void shouldCloseCleanRestoredTask() {
.withInputPartitions(taskId00Partitions).build();
final TasksRegistry tasks = mock(TasksRegistry.class);
final TaskManager taskManager = setUpCloseCleanRestoredTask(statefulTask, tasks);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(statefulTask.id());
replay(activeTaskCreator);
taskManager.checkStateUpdater(time.milliseconds(), noOpResetter);
@@ -1152,7 +1140,6 @@ public void shouldHandleExceptionThrownDuringCloseInCloseCleanRestoredTask() {
final TasksRegistry tasks = mock(TasksRegistry.class);
final TaskManager taskManager = setUpCloseCleanRestoredTask(statefulTask, tasks);
doThrow(RuntimeException.class).when(statefulTask).closeClean();
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(statefulTask.id());
replay(activeTaskCreator);
assertThrows(
@@ -1165,27 +1152,6 @@ public void shouldHandleExceptionThrownDuringCloseInCloseCleanRestoredTask() {
Mockito.verify(tasks, never()).removeTask(statefulTask);
}
- @Test
- public void shouldHandleExceptionThrownDuringClosingTaskProducerInCloseCleanRestoredTask() {
- final StreamTask statefulTask = statefulTask(taskId00, taskId00ChangelogPartitions)
- .inState(State.CLOSED)
- .withInputPartitions(taskId00Partitions).build();
- final TasksRegistry tasks = mock(TasksRegistry.class);
- final TaskManager taskManager = setUpCloseCleanRestoredTask(statefulTask, tasks);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(statefulTask.id());
- expectLastCall().andThrow(new RuntimeException("Something happened"));
- replay(activeTaskCreator);
-
- assertThrows(
- RuntimeException.class,
- () -> taskManager.checkStateUpdater(time.milliseconds(), noOpResetter)
- );
-
- verify(activeTaskCreator);
- Mockito.verify(statefulTask, never()).closeDirty();
- Mockito.verify(tasks, never()).removeTask(statefulTask);
- }
-
private TaskManager setUpCloseCleanRestoredTask(final StreamTask statefulTask,
final TasksRegistry tasks) {
when(tasks.removePendingTaskToRecycle(statefulTask.id())).thenReturn(null);
@@ -1206,7 +1172,6 @@ public void shouldCloseDirtyRestoredTask() {
when(tasks.removePendingTaskToCloseDirty(statefulTask.id())).thenReturn(true);
when(stateUpdater.drainRestoredActiveTasks(any(Duration.class))).thenReturn(mkSet(statefulTask));
when(stateUpdater.restoresActiveTasks()).thenReturn(true);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(statefulTask.id());
final TaskManager taskManager = setUpTaskManager(ProcessingMode.AT_LEAST_ONCE, tasks, true);
replay(activeTaskCreator);
@@ -1670,8 +1635,6 @@ public void shouldCloseActiveUnassignedSuspendedTasksWhenClosingRevokedTasks() {
expectRestoreToBeCompleted(consumer, changeLogReader);
expect(activeTaskCreator.createTasks(anyObject(), eq(taskId00Assignment))).andStubReturn(singletonList(task00));
expect(activeTaskCreator.createTasks(anyObject(), eq(emptyMap()))).andStubReturn(emptyList());
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall();
expect(standbyTaskCreator.createTasks(anyObject())).andStubReturn(emptyList());
topologyBuilder.addSubscribedTopicsFromAssignment(anyObject(), anyString());
expectLastCall().anyTimes();
@@ -1712,8 +1675,6 @@ public void closeClean() {
expectRestoreToBeCompleted(consumer, changeLogReader);
expect(activeTaskCreator.createTasks(anyObject(), eq(taskId00Assignment))).andStubReturn(singletonList(task00));
expect(activeTaskCreator.createTasks(anyObject(), eq(emptyMap()))).andStubReturn(emptyList());
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall();
expect(standbyTaskCreator.createTasks(anyObject())).andStubReturn(emptyList());
topologyBuilder.addSubscribedTopicsFromAssignment(anyObject(), anyString());
expectLastCall().anyTimes();
@@ -1759,10 +1720,6 @@ public void shouldCloseActiveTasksWhenHandlingLostTasks() throws Exception {
taskManager.handleRebalanceStart(emptySet());
assertThat(taskManager.lockedTaskDirectories(), Matchers.is(mkSet(taskId00, taskId01)));
- // `handleLostAll`
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall();
-
replay(activeTaskCreator, standbyTaskCreator, topologyBuilder, consumer, changeLogReader);
taskManager.handleAssignment(taskId00Assignment, taskId01Assignment);
@@ -1799,46 +1756,6 @@ public void shouldReInitializeThreadProducerOnHandleLostAllIfEosV2Enabled() {
verify(activeTaskCreator);
}
- @Test
- public void shouldThrowWhenHandlingClosingTasksOnProducerCloseError() {
- final StateMachineTask task00 = new StateMachineTask(taskId00, taskId00Partitions, true);
- final Map offsets = singletonMap(t1p0, new OffsetAndMetadata(0L, null));
- task00.setCommittableOffsetsAndMetadata(offsets);
-
- // `handleAssignment`
- expectRestoreToBeCompleted(consumer, changeLogReader);
- expect(activeTaskCreator.createTasks(anyObject(), eq(taskId00Assignment))).andStubReturn(singletonList(task00));
- expect(standbyTaskCreator.createTasks(anyObject())).andStubReturn(emptyList());
- topologyBuilder.addSubscribedTopicsFromAssignment(anyObject(), anyString());
- expectLastCall().anyTimes();
-
- // `handleAssignment`
- consumer.commitSync(offsets);
- expectLastCall();
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall().andThrow(new RuntimeException("KABOOM!"));
-
- replay(activeTaskCreator, standbyTaskCreator, topologyBuilder, consumer, changeLogReader);
-
- taskManager.handleAssignment(taskId00Assignment, emptyMap());
- assertThat(taskManager.tryToCompleteRestoration(time.milliseconds(), null), is(true));
- assertThat(task00.state(), is(Task.State.RUNNING));
-
- taskManager.handleRevocation(taskId00Partitions);
-
- final RuntimeException thrown = assertThrows(
- RuntimeException.class,
- () -> taskManager.handleAssignment(emptyMap(), emptyMap())
- );
-
- assertThat(
- thrown.getMessage(),
- is("Encounter unexpected fatal error for task 0_0")
- );
- assertThat(thrown.getCause(), instanceOf(RuntimeException.class));
- assertThat(thrown.getCause().getMessage(), is("KABOOM!"));
- }
-
@Test
public void shouldReviveCorruptTasks() {
final ProcessorStateManager stateManager = EasyMock.createStrictMock(ProcessorStateManager.class);
@@ -2280,8 +2197,6 @@ public void markChangelogAsCorrupted(final Collection partitions
expect(activeTaskCreator.createTasks(anyObject(), eq(assignmentActive))).andReturn(asList(revokedActiveTask, unrevokedActiveTaskWithCommitNeeded, unrevokedActiveTaskWithoutCommitNeeded));
expect(standbyTaskCreator.createTasks(anyObject())).andStubReturn(Collections.emptySet());
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall();
consumer.commitSync(expectedCommittedOffsets);
expectLastCall().andThrow(new TimeoutException());
expect(consumer.assignment()).andStubReturn(union(HashSet::new, taskId00Partitions, taskId01Partitions, taskId02Partitions));
@@ -2344,8 +2259,6 @@ public void markChangelogAsCorrupted(final Collection partitions
expect(activeTaskCreator.createTasks(anyObject(), eq(assignmentActive))).andReturn(asList(revokedActiveTask, unrevokedActiveTask, unrevokedActiveTaskWithoutCommitNeeded));
expect(standbyTaskCreator.createTasks(anyObject())).andStubReturn(Collections.emptySet());
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall();
final ConsumerGroupMetadata groupMetadata = new ConsumerGroupMetadata("appId");
expect(consumer.groupMetadata()).andReturn(groupMetadata);
@@ -2627,7 +2540,6 @@ public void shouldCommitAllActiveTasksThatNeedCommittingOnHandleRevocationWithEo
.andReturn(asList(task00, task01, task02));
expect(activeTaskCreator.threadProducer()).andReturn(producer);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
expect(standbyTaskCreator.createTasks(eq(assignmentStandby)))
.andReturn(singletonList(task10));
@@ -2697,8 +2609,6 @@ public void shouldCommitAllNeededTasksOnHandleRevocation() {
expect(activeTaskCreator.createTasks(anyObject(), eq(assignmentActive)))
.andReturn(asList(task00, task01, task02));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall();
expect(standbyTaskCreator.createTasks(eq(assignmentStandby)))
.andReturn(singletonList(task10));
consumer.commitSync(expectedCommittedOffsets);
@@ -2794,8 +2704,6 @@ public void shouldNotCommitCreatedTasksOnRevocationOrClosure() {
expect(activeTaskCreator.createTasks(anyObject(), eq(taskId00Assignment))).andReturn(singletonList(task00));
expect(standbyTaskCreator.createTasks(anyObject())).andStubReturn(Collections.emptySet());
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(eq(taskId00));
- expectLastCall().once();
expect(activeTaskCreator.createTasks(anyObject(), eq(Collections.emptyMap()))).andReturn(Collections.emptySet());
replay(activeTaskCreator, standbyTaskCreator, consumer, changeLogReader);
@@ -2896,8 +2804,6 @@ public void closeDirty() {
expect(changeLogReader.completedChangelogs()).andReturn(emptySet());
expect(activeTaskCreator.createTasks(anyObject(), eq(assignment)))
.andStubReturn(asList(task00, task01, task02, task03));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(anyObject());
- expectLastCall().times(4);
activeTaskCreator.closeThreadProducerIfNeeded();
expectLastCall();
expect(standbyTaskCreator.createTasks(eq(emptyMap()))).andStubReturn(emptyList());
@@ -2948,59 +2854,6 @@ public void closeDirty() {
verify(activeTaskCreator, changeLogReader);
}
- @Test
- public void shouldCloseActiveTasksAndPropagateTaskProducerExceptionsOnCleanShutdown() {
- final TopicPartition changelog = new TopicPartition("changelog", 0);
- final Map> assignment = mkMap(
- mkEntry(taskId00, taskId00Partitions)
- );
- final StateMachineTask task00 = new StateMachineTask(taskId00, taskId00Partitions, true) {
- @Override
- public Set changelogPartitions() {
- return singleton(changelog);
- }
- };
- final Map offsets = singletonMap(t1p0, new OffsetAndMetadata(0L, null));
- task00.setCommittableOffsetsAndMetadata(offsets);
-
- resetToStrict(changeLogReader);
- changeLogReader.enforceRestoreActive();
- expect(changeLogReader.completedChangelogs()).andReturn(emptySet());
- expect(activeTaskCreator.createTasks(anyObject(), eq(assignment))).andStubReturn(singletonList(task00));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(eq(taskId00));
- expectLastCall().andThrow(new RuntimeException("whatever"));
- activeTaskCreator.closeThreadProducerIfNeeded();
- expectLastCall();
- expect(standbyTaskCreator.createTasks(eq(emptyMap()))).andStubReturn(emptyList());
- replay(activeTaskCreator, standbyTaskCreator, changeLogReader);
-
- taskManager.handleAssignment(assignment, emptyMap());
-
- assertThat(task00.state(), is(Task.State.CREATED));
-
- taskManager.tryToCompleteRestoration(time.milliseconds(), null);
-
- assertThat(task00.state(), is(Task.State.RESTORING));
- assertThat(
- taskManager.activeTaskMap(),
- Matchers.equalTo(
- mkMap(
- mkEntry(taskId00, task00)
- )
- )
- );
- assertThat(taskManager.standbyTaskMap(), Matchers.anEmptyMap());
-
- final RuntimeException exception = assertThrows(RuntimeException.class, () -> taskManager.shutdown(true));
-
- assertThat(task00.state(), is(Task.State.CLOSED));
- assertThat(exception.getCause().getMessage(), is("whatever"));
- assertThat(taskManager.activeTaskMap(), Matchers.anEmptyMap());
- assertThat(taskManager.standbyTaskMap(), Matchers.anEmptyMap());
- // the active task creator should also get closed (so that it closes the thread producer if applicable)
- verify(activeTaskCreator, changeLogReader);
- }
-
@Test
public void shouldCloseActiveTasksAndPropagateThreadProducerExceptionsOnCleanShutdown() {
final TopicPartition changelog = new TopicPartition("changelog", 0);
@@ -3018,8 +2871,6 @@ public Set changelogPartitions() {
changeLogReader.enforceRestoreActive();
expect(changeLogReader.completedChangelogs()).andReturn(emptySet());
expect(activeTaskCreator.createTasks(anyObject(), eq(assignment))).andStubReturn(singletonList(task00));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(eq(taskId00));
- expectLastCall();
activeTaskCreator.closeThreadProducerIfNeeded();
expectLastCall().andThrow(new RuntimeException("whatever"));
expect(standbyTaskCreator.createTasks(eq(emptyMap()))).andStubReturn(emptyList());
@@ -3054,7 +2905,7 @@ public Set changelogPartitions() {
@Test
public void shouldOnlyCommitRevokedStandbyTaskAndPropagatePrepareCommitException() {
- setUpTaskManager(ProcessingMode.EXACTLY_ONCE_ALPHA, false);
+ setUpTaskManager(ProcessingMode.EXACTLY_ONCE_V2, false);
final Task task00 = new StateMachineTask(taskId00, taskId00Partitions, false);
@@ -3147,8 +2998,6 @@ public void suspend() {
changeLogReader.enforceRestoreActive();
expect(changeLogReader.completedChangelogs()).andReturn(emptySet());
expect(activeTaskCreator.createTasks(anyObject(), eq(assignment))).andStubReturn(asList(task00, task01, task02));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(anyObject());
- expectLastCall().andThrow(new RuntimeException("whatever")).times(3);
activeTaskCreator.closeThreadProducerIfNeeded();
expectLastCall().andThrow(new RuntimeException("whatever all"));
expect(standbyTaskCreator.createTasks(eq(emptyMap()))).andStubReturn(emptyList());
@@ -3239,7 +3088,6 @@ public void shouldShutDownStateUpdaterAndCloseFailedTasksDirty() {
new ExceptionAndTasks(mkSet(failedStatefulTask), new RuntimeException()),
new ExceptionAndTasks(mkSet(failedStandbyTask), new RuntimeException()))
);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(failedStatefulTask.id());
activeTaskCreator.closeThreadProducerIfNeeded();
final TaskManager taskManager = setUpTaskManager(ProcessingMode.AT_LEAST_ONCE, tasks, true);
replay(activeTaskCreator);
@@ -3264,8 +3112,6 @@ public void shouldShutDownStateUpdaterAndAddRestoredTasksToTaskRegistry() {
final Set restoredTasks = restoredActiveTasks.stream().map(t -> (Task) t).collect(Collectors.toSet());
when(stateUpdater.drainRestoredActiveTasks(Duration.ZERO)).thenReturn(restoredActiveTasks);
when(tasks.activeTasks()).thenReturn(restoredTasks);
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(statefulTask1.id());
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(statefulTask2.id());
activeTaskCreator.closeThreadProducerIfNeeded();
final TaskManager taskManager = setUpTaskManager(ProcessingMode.AT_LEAST_ONCE, tasks, true);
replay(activeTaskCreator);
@@ -3289,7 +3135,6 @@ public void shouldShutDownStateUpdaterAndAddRemovedTasksToTaskRegistry() {
when(stateUpdater.drainRemovedTasks()).thenReturn(mkSet(removedStandbyTask, removedStatefulTask));
when(tasks.activeTasks()).thenReturn(mkSet(removedStatefulTask));
when(tasks.allTasks()).thenReturn(mkSet(removedStatefulTask, removedStandbyTask));
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(removedStatefulTask.id());
activeTaskCreator.closeThreadProducerIfNeeded();
final TaskManager taskManager = setUpTaskManager(ProcessingMode.AT_LEAST_ONCE, tasks, true);
replay(activeTaskCreator);
@@ -3513,24 +3358,6 @@ public void shouldCommitViaConsumerIfEosDisabled() {
verify(consumer);
}
- @Test
- public void shouldCommitViaProducerIfEosAlphaEnabled() {
- final StreamsProducer producer = EasyMock.mock(StreamsProducer.class);
- expect(activeTaskCreator.streamsProducerForTask(anyObject(TaskId.class)))
- .andReturn(producer)
- .andReturn(producer);
-
- final Map offsetsT01 = singletonMap(t1p1, new OffsetAndMetadata(0L, null));
- final Map offsetsT02 = singletonMap(t1p2, new OffsetAndMetadata(1L, null));
-
- producer.commitTransaction(offsetsT01, new ConsumerGroupMetadata("appId"));
- expectLastCall();
- producer.commitTransaction(offsetsT02, new ConsumerGroupMetadata("appId"));
- expectLastCall();
-
- shouldCommitViaProducerIfEosEnabled(ProcessingMode.EXACTLY_ONCE_ALPHA, producer, offsetsT01, offsetsT02);
- }
-
@Test
public void shouldCommitViaProducerIfEosV2Enabled() {
final StreamsProducer producer = EasyMock.mock(StreamsProducer.class);
@@ -4323,50 +4150,6 @@ public void shouldNotFailForTimeoutExceptionOnConsumerCommit() {
assertNull(task01.timeout);
}
- @Test
- public void shouldNotFailForTimeoutExceptionOnCommitWithEosAlpha() {
- final TaskManager taskManager = setUpTaskManager(ProcessingMode.EXACTLY_ONCE_ALPHA, false);
-
- final StreamsProducer producer = mock(StreamsProducer.class);
- expect(activeTaskCreator.streamsProducerForTask(anyObject(TaskId.class)))
- .andReturn(producer)
- .andReturn(producer)
- .andReturn(producer);
-
- final Map offsetsT00 = singletonMap(t1p0, new OffsetAndMetadata(0L, null));
- final Map offsetsT01 = singletonMap(t1p1, new OffsetAndMetadata(1L, null));
-
- doThrow(new TimeoutException("KABOOM!"))
- .doNothing()
- .doNothing()
- .doNothing()
- .when(producer).commitTransaction(offsetsT00, null);
- doNothing()
- .doNothing()
- .when(producer).commitTransaction(offsetsT01, null);
-
- final StateMachineTask task00 = new StateMachineTask(taskId00, taskId00Partitions, true);
- task00.setCommittableOffsetsAndMetadata(offsetsT00);
- final StateMachineTask task01 = new StateMachineTask(taskId01, taskId01Partitions, true);
- task01.setCommittableOffsetsAndMetadata(offsetsT01);
- final StateMachineTask task02 = new StateMachineTask(taskId02, taskId02Partitions, true);
-
- expect(consumer.groupMetadata()).andStubReturn(null);
- replay(activeTaskCreator, consumer);
-
- task00.setCommitNeeded();
- task01.setCommitNeeded();
-
- final TaskCorruptedException exception = assertThrows(
- TaskCorruptedException.class,
- () -> taskManager.commit(mkSet(task00, task01, task02))
- );
- assertThat(
- exception.corruptedTasks(),
- equalTo(Collections.singleton(taskId00))
- );
- }
-
@Test
public void shouldThrowTaskCorruptedExceptionForTimeoutExceptionOnCommitWithEosV2() {
final TaskManager taskManager = setUpTaskManager(ProcessingMode.EXACTLY_ONCE_V2, false);
@@ -4491,8 +4274,6 @@ public void shouldConvertActiveTaskToStandbyTask() {
expect(standbyTaskCreator.createTasks(anyObject())).andStubReturn(Collections.emptySet());
activeTask.prepareRecycle();
expectLastCall().once();
- activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
- expectLastCall().anyTimes();
expect(standbyTaskCreator.createStandbyTaskFromActive(anyObject(), eq(taskId00Partitions))).andReturn(standbyTask);
expect(activeTaskCreator.createTasks(anyObject(), eq(Collections.emptyMap()))).andReturn(Collections.emptySet());
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/KeyValueStoreTestDriver.java b/streams/src/test/java/org/apache/kafka/streams/state/KeyValueStoreTestDriver.java
index 178399426076..76747db5ca5a 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/KeyValueStoreTestDriver.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/KeyValueStoreTestDriver.java
@@ -217,7 +217,6 @@ private KeyValueStoreTestDriver(final StateSerdes serdes) {
"threadId",
new MockClientSupplier(),
null,
- null,
logContext,
Time.SYSTEM),
new DefaultProductionExceptionHandler(),
diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
index a70bf5781481..278be31b3841 100644
--- a/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/StreamThreadStateStoreProviderTest.java
@@ -424,7 +424,6 @@ private StreamTask createStreamsTask(final StreamsConfig streamsConfig,
streamsConfig,
"threadId",
clientSupplier,
- new TaskId(0, 0),
UUID.randomUUID(),
logContext,
Time.SYSTEM
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java
index 8a402ab9abf5..5458259a9645 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/BrokerCompatibilityTest.java
@@ -104,7 +104,7 @@ public static void main(final String[] args) throws IOException {
System.out.println("start Kafka Streams");
streams.start();
- final boolean eosEnabled = processingMode.startsWith("exactly_once");
+ final boolean eosEnabled = processingMode.equals("exactly_once_v2");
System.out.println("send data");
final Properties producerProperties = new Properties();
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsEosTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsEosTest.java
index 5ad0641b60fb..62223d789f03 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsEosTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsEosTest.java
@@ -29,7 +29,6 @@ public class StreamsEosTest {
* args ::= kafka propFileName command
* command := "run" | "process" | "verify"
*/
- @SuppressWarnings("deprecation")
public static void main(final String[] args) throws IOException {
if (args.length < 2) {
System.err.println("StreamsEosTest are expecting two parameters: propFile, command; but only see " + args.length + " parameter");
@@ -49,12 +48,9 @@ public static void main(final String[] args) throws IOException {
}
if ("process".equals(command) || "process-complex".equals(command)) {
- if (!StreamsConfig.EXACTLY_ONCE.equals(processingGuarantee) &&
- !StreamsConfig.EXACTLY_ONCE_BETA.equals(processingGuarantee) &&
- !StreamsConfig.EXACTLY_ONCE_V2.equals(processingGuarantee)) {
+ if (!StreamsConfig.EXACTLY_ONCE_V2.equals(processingGuarantee)) {
- System.err.println("processingGuarantee must be either " + StreamsConfig.EXACTLY_ONCE + " or " +
- StreamsConfig.EXACTLY_ONCE_BETA + " or " + StreamsConfig.EXACTLY_ONCE_V2);
+ System.err.println("processingGuarantee must be " + StreamsConfig.EXACTLY_ONCE_V2);
Exit.exit(1);
}
}
diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java
index d87da749ee9a..c1db8c9b8302 100644
--- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java
@@ -38,7 +38,6 @@ public class StreamsSmokeTest {
*
* @param args
*/
- @SuppressWarnings("deprecation")
public static void main(final String[] args) throws IOException {
if (args.length < 2) {
System.err.println("StreamsSmokeTest are expecting two parameters: propFile, command; but only see " + args.length + " parameter");
@@ -60,15 +59,11 @@ public static void main(final String[] args) throws IOException {
if ("process".equals(command)) {
if (!StreamsConfig.AT_LEAST_ONCE.equals(processingGuarantee) &&
- !StreamsConfig.EXACTLY_ONCE.equals(processingGuarantee) &&
- !StreamsConfig.EXACTLY_ONCE_BETA.equals(processingGuarantee) &&
!StreamsConfig.EXACTLY_ONCE_V2.equals(processingGuarantee)) {
System.err.println("processingGuarantee must be either " +
- StreamsConfig.AT_LEAST_ONCE + ", " +
- StreamsConfig.EXACTLY_ONCE + ", or " +
- StreamsConfig.EXACTLY_ONCE_BETA + ", or " +
- StreamsConfig.EXACTLY_ONCE_V2);
+ StreamsConfig.AT_LEAST_ONCE + " or " +
+ StreamsConfig.EXACTLY_ONCE_V2);
Exit.exit(1);
}
diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
index 9e7e285bd10b..85d2c1c4a3ba 100644
--- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
+++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java
@@ -115,7 +115,6 @@
import java.util.regex.Pattern;
import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.AT_LEAST_ONCE;
-import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_ALPHA;
import static org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_V2;
import static org.apache.kafka.streams.state.ValueAndTimestamp.getValueOrNull;
@@ -468,7 +467,6 @@ private void setupGlobalTask(final Time mockWallClockTime,
}
}
- @SuppressWarnings("deprecation")
private void setupTask(final StreamsConfig streamsConfig,
final StreamsMetricsImpl streamsMetrics,
final ThreadCache cache,
@@ -484,7 +482,7 @@ private void setupTask(final StreamsConfig streamsConfig,
final ProcessorStateManager stateManager = new ProcessorStateManager(
TASK_ID,
Task.TaskType.ACTIVE,
- StreamsConfig.EXACTLY_ONCE.equals(streamsConfig.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG)),
+ StreamsConfig.EXACTLY_ONCE_V2.equals(streamsConfig.getString(StreamsConfig.PROCESSING_GUARANTEE_CONFIG)),
logContext,
stateDirectory,
new MockChangelogRegister(),
@@ -614,7 +612,7 @@ private void completeAllProcessableWork() {
}
private void commit(final Map offsets) {
- if (processingMode == EXACTLY_ONCE_ALPHA || processingMode == EXACTLY_ONCE_V2) {
+ if (processingMode == EXACTLY_ONCE_V2) {
testDriverProducer.commitTransaction(offsets, new ConsumerGroupMetadata("dummy-app-id"));
} else {
consumer.commitSync(offsets);
@@ -1351,7 +1349,7 @@ public TestDriverProducer(final StreamsConfig config,
final KafkaClientSupplier clientSupplier,
final LogContext logContext,
final Time time) {
- super(config, "TopologyTestDriver-StreamThread-1", clientSupplier, new TaskId(0, 0), UUID.randomUUID(), logContext, time);
+ super(config, "TopologyTestDriver-StreamThread-1", clientSupplier, UUID.randomUUID(), logContext, time);
}
@Override
diff --git a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
index b1f6dfe1ab94..753eb95d7269 100644
--- a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
+++ b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py
@@ -30,7 +30,6 @@ class StreamsBrokerCompatibility(Test):
"""
These tests validates that
- Streams works for older brokers 0.11 (or newer)
- - Streams w/ EOS-alpha works for older brokers 0.11 (or newer)
- Streams w/ EOS-v2 works for older brokers 2.5 (or newer)
- Streams fails fast for older brokers 0.10.0, 0.10.2, and 0.10.1
- Streams w/ EOS-v2 fails fast for older brokers 2.4 or older
@@ -95,38 +94,6 @@ def test_compatible_brokers_eos_disabled(self, broker_version):
self.consumer.stop()
self.kafka.stop()
- @cluster(num_nodes=4)
- @parametrize(broker_version=str(LATEST_3_2))
- @parametrize(broker_version=str(LATEST_3_1))
- @parametrize(broker_version=str(LATEST_3_0))
- @parametrize(broker_version=str(LATEST_2_8))
- @parametrize(broker_version=str(LATEST_2_7))
- @parametrize(broker_version=str(LATEST_2_6))
- @parametrize(broker_version=str(LATEST_2_5))
- @parametrize(broker_version=str(LATEST_2_4))
- @parametrize(broker_version=str(LATEST_2_3))
- @parametrize(broker_version=str(LATEST_2_2))
- @parametrize(broker_version=str(LATEST_2_1))
- @parametrize(broker_version=str(LATEST_2_0))
- @parametrize(broker_version=str(LATEST_1_1))
- @parametrize(broker_version=str(LATEST_1_0))
- @parametrize(broker_version=str(LATEST_0_11_0))
- def test_compatible_brokers_eos_alpha_enabled(self, broker_version):
- self.kafka.set_version(KafkaVersion(broker_version))
- self.kafka.start()
-
- processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, "exactly_once")
- processor.start()
-
- self.consumer.start()
-
- processor.wait()
-
- wait_until(lambda: self.consumer.total_consumed() > 0, timeout_sec=30, err_msg="Did expect to read a message but got none within 30 seconds.")
-
- self.consumer.stop()
- self.kafka.stop()
-
@cluster(num_nodes=4)
@parametrize(broker_version=str(LATEST_3_2))
@parametrize(broker_version=str(LATEST_3_1))
@@ -187,9 +154,9 @@ def test_fail_fast_on_incompatible_brokers_if_eos_v2_enabled(self, broker_versio
with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
with processor.node.account.monitor_log(processor.LOG_FILE) as log:
processor.start()
- log.wait_until('Shutting down because the Kafka cluster seems to be on a too old version. Setting processing\.guarantee="exactly_once_v2"/"exactly_once_beta" requires broker version 2\.5 or higher\.',
+ log.wait_until('Shutting down because the Kafka cluster seems to be on a too old version. Setting processing\.guarantee="exactly_once_v2" requires broker version 2\.5 or higher\.',
timeout_sec=60,
- err_msg="Never saw 'Shutting down because the Kafka cluster seems to be on a too old version. Setting `processing.guarantee=\"exactly_once_v2\"/\"exaclty_once_beta\"` requires broker version 2.5 or higher.' log message " + str(processor.node.account))
+ err_msg="Never saw 'Shutting down because the Kafka cluster seems to be on a too old version. Setting processing.guarantee=\"exactly_once_v2\" requires broker version 2.5 or higher.' log message " + str(processor.node.account))
monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException',
timeout_sec=60,
err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException' error message " + str(processor.node.account))
diff --git a/tests/kafkatest/tests/streams/streams_eos_test.py b/tests/kafkatest/tests/streams/streams_eos_test.py
index 8d8853ac461a..01c5d27bf61d 100644
--- a/tests/kafkatest/tests/streams/streams_eos_test.py
+++ b/tests/kafkatest/tests/streams/streams_eos_test.py
@@ -39,21 +39,19 @@ def __init__(self, test_context):
self.test_context = test_context
@cluster(num_nodes=9)
- @matrix(processing_guarantee=["exactly_once", "exactly_once_v2"],
- metadata_quorum=[quorum.remote_kraft])
- def test_rebalance_simple(self, processing_guarantee, metadata_quorum):
- self.run_rebalance(StreamsEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
+ @matrix(metadata_quorum=[quorum.remote_kraft])
+ def test_rebalance_simple(self, metadata_quorum):
+ self.run_rebalance(StreamsEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
StreamsEosTestVerifyRunnerService(self.test_context, self.kafka))
@cluster(num_nodes=9)
- @matrix(processing_guarantee=["exactly_once", "exactly_once_v2"],
- metadata_quorum=[quorum.remote_kraft])
- def test_rebalance_complex(self, processing_guarantee, metadata_quorum):
- self.run_rebalance(StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
+ @matrix(metadata_quorum=[quorum.remote_kraft])
+ def test_rebalance_complex(self, metadata_quorum):
+ self.run_rebalance(StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
StreamsComplexEosTestVerifyRunnerService(self.test_context, self.kafka))
def run_rebalance(self, processor1, processor2, processor3, verifier):
@@ -83,21 +81,19 @@ def run_rebalance(self, processor1, processor2, processor3, verifier):
verifier.node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % verifier.STDOUT_FILE, allow_fail=False)
@cluster(num_nodes=9)
- @matrix(processing_guarantee=["exactly_once", "exactly_once_v2"],
- metadata_quorum=[quorum.remote_kraft])
- def test_failure_and_recovery(self, processing_guarantee, metadata_quorum):
- self.run_failure_and_recovery(StreamsEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
+ @matrix(metadata_quorum=[quorum.remote_kraft])
+ def test_failure_and_recovery(self, metadata_quorum):
+ self.run_failure_and_recovery(StreamsEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
StreamsEosTestVerifyRunnerService(self.test_context, self.kafka))
@cluster(num_nodes=9)
- @matrix(processing_guarantee=["exactly_once", "exactly_once_v2"],
- metadata_quorum=[quorum.remote_kraft])
- def test_failure_and_recovery_complex(self, processing_guarantee, metadata_quorum):
- self.run_failure_and_recovery(StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
- StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, processing_guarantee),
+ @matrix(metadata_quorum=[quorum.remote_kraft])
+ def test_failure_and_recovery_complex(self, metadata_quorum):
+ self.run_failure_and_recovery(StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
+ StreamsComplexEosTestJobRunnerService(self.test_context, self.kafka, "exactly_once_v2"),
StreamsComplexEosTestVerifyRunnerService(self.test_context, self.kafka))
def run_failure_and_recovery(self, processor1, processor2, processor3, verifier):
diff --git a/tests/kafkatest/tests/streams/streams_relational_smoke_test.py b/tests/kafkatest/tests/streams/streams_relational_smoke_test.py
index b91ec0c3e608..b8485ce17f29 100644
--- a/tests/kafkatest/tests/streams/streams_relational_smoke_test.py
+++ b/tests/kafkatest/tests/streams/streams_relational_smoke_test.py
@@ -86,15 +86,14 @@ def __init__(self, test_context):
@cluster(num_nodes=8)
@matrix(crash=[False, True],
- processing_guarantee=['exactly_once', 'exactly_once_v2'],
metadata_quorum=[quorum.remote_kraft])
- def test_streams(self, crash, processing_guarantee, metadata_quorum):
+ def test_streams(self, crash, metadata_quorum):
driver = StreamsRelationalSmokeTestService(self.test_context, self.kafka, "driver", "ignored", "ignored")
LOG_FILE = driver.LOG_FILE # this is the same for all instances of the service, so we can just declare a "constant"
- processor1 = StreamsRelationalSmokeTestService(self.test_context, self.kafka, "application", "processor1", processing_guarantee)
- processor2 = StreamsRelationalSmokeTestService(self.test_context, self.kafka, "application", "processor2", processing_guarantee)
+ processor1 = StreamsRelationalSmokeTestService(self.test_context, self.kafka, "application", "processor1", 'exactly_once_v2')
+ processor2 = StreamsRelationalSmokeTestService(self.test_context, self.kafka, "application", "processor2", 'exactly_once_v2')
processor1.start()
processor2.start()
@@ -109,7 +108,7 @@ def test_streams(self, crash, processing_guarantee, metadata_quorum):
processor1.stop_nodes(not crash)
- processor3 = StreamsRelationalSmokeTestService(self.test_context, self.kafka, "application", "processor3", processing_guarantee)
+ processor3 = StreamsRelationalSmokeTestService(self.test_context, self.kafka, "application", "processor3", 'exactly_once_v2')
processor3.start()
processor3.await_command("grep -q 'Streams has started' %s" % LOG_FILE)
diff --git a/tests/kafkatest/tests/streams/streams_smoke_test.py b/tests/kafkatest/tests/streams/streams_smoke_test.py
index 8a119f37d990..88846de89491 100644
--- a/tests/kafkatest/tests/streams/streams_smoke_test.py
+++ b/tests/kafkatest/tests/streams/streams_smoke_test.py
@@ -47,7 +47,7 @@ def __init__(self, test_context):
self.driver = StreamsSmokeTestDriverService(test_context, self.kafka)
@cluster(num_nodes=8)
- @matrix(processing_guarantee=['exactly_once', 'exactly_once_v2', 'at_least_once'],
+ @matrix(processing_guarantee=['exactly_once_v2', 'at_least_once'],
crash=[True, False],
metadata_quorum=quorum.all_non_upgrade)
def test_streams(self, processing_guarantee, crash, metadata_quorum=quorum.zk):