Skip to content

Commit

Permalink
Merge commit 'c2f6f29ca6e1306ac77ec726bac4cd09bd1aa80b' into sync-ups…
Browse files Browse the repository at this point in the history 
…tream-3.5

* commit 'c2f6f29ca6e1306ac77ec726bac4cd09bd1aa80b': (76 commits)
  KAFKA-15019: Improve handling of broker heartbeat timeouts (apache#13759)
  KAFKA-15003: Fix ZK sync logic for partition assignments (apache#13735)
  MINOR: Add 3.5 upgrade steps for ZK and KRaft (apache#13792)
  KAFKA-15010 ZK migration failover support (apache#13758)
  KAFKA-15017 Fix snapshot load in dual write mode for ClientQuotas and SCRAM  (apache#13757)
  MINOR: Update LICENSE-binary following snappy upgrade (apache#13791)
  Upgrade to snappy v1.1.10.0 (apache#13786)
  KAFKA-15004: Fix configuration dual-write during migration (apache#13767)
  KAFKA-8713: JsonConverter replace.null.with.default should prevent emitting default for Struct fields (apache#13781)
  KAFKA-14996: Handle overly large user operations on the kcontroller (apache#13742)
  ...
  • Loading branch information
@emissionnebula
emissionnebula committed Jun 16, 2023
2 parents b47b063 + c2f6f29 commit d14b738
Show file tree
Hide file tree
Showing 352 changed files with 13,920 additions and 6,546 deletions.
102 changes: 48 additions & 54 deletions LICENSE-binary
View file
Expand Up @@ -205,64 +205,55 @@
This project bundles some components that are also licensed under the Apache
License Version 2.0:

audience-annotations-0.5.0
audience-annotations-0.13.0
commons-cli-1.4
commons-lang3-3.8.1
jackson-annotations-2.13.4
jackson-core-2.13.4
jackson-databind-2.13.4.2
jackson-dataformat-csv-2.13.4
jackson-dataformat-yaml-2.13.4
jackson-datatype-jdk8-2.13.4
jackson-datatype-jsr310-2.13.4
jackson-jaxrs-base-2.13.4
jackson-jaxrs-json-provider-2.13.4
jackson-module-jaxb-annotations-2.13.4
jackson-module-scala_2.13-2.13.4
jackson-module-scala_2.12-2.13.4
jackson-annotations-2.13.5
jackson-core-2.13.5
jackson-databind-2.13.5
jackson-dataformat-csv-2.13.5
jackson-datatype-jdk8-2.13.5
jackson-jaxrs-base-2.13.5
jackson-jaxrs-json-provider-2.13.5
jackson-module-jaxb-annotations-2.13.5
jackson-module-scala_2.13-2.13.5
jackson-module-scala_2.12-2.13.5
jakarta.validation-api-2.0.2
javassist-3.27.0-GA
jetty-client-9.4.48.v20220622
jetty-continuation-9.4.48.v20220622
jetty-http-9.4.48.v20220622
jetty-io-9.4.48.v20220622
jetty-security-9.4.48.v20220622
jetty-server-9.4.48.v20220622
jetty-servlet-9.4.48.v20220622
jetty-servlets-9.4.48.v20220622
jetty-util-9.4.48.v20220622
jetty-util-ajax-9.4.48.v20220622
jersey-common-2.34
jersey-server-2.34
jose4j-0.7.9
javassist-3.29.2-GA
jetty-client-9.4.51.v20230217
jetty-continuation-9.4.51.v20230217
jetty-http-9.4.51.v20230217
jetty-io-9.4.51.v20230217
jetty-security-9.4.51.v20230217
jetty-server-9.4.51.v20230217
jetty-servlet-9.4.51.v20230217
jetty-servlets-9.4.51.v20230217
jetty-util-9.4.51.v20230217
jetty-util-ajax-9.4.51.v20230217
jose4j-0.9.3
lz4-java-1.8.0
maven-artifact-3.8.4
metrics-core-4.1.12.1
metrics-core-2.2.0
netty-buffer-4.1.78.Final
netty-codec-4.1.78.Final
netty-common-4.1.78.Final
netty-handler-4.1.78.Final
netty-resolver-4.1.78.Final
netty-transport-4.1.78.Final
netty-transport-classes-epoll-4.1.78.Final
netty-transport-native-epoll-4.1.78.Final
netty-transport-native-unix-common-4.1.78.Final
netty-buffer-4.1.92.Final
netty-codec-4.1.92.Final
netty-common-4.1.92.Final
netty-handler-4.1.92.Final
netty-resolver-4.1.92.Final
netty-transport-4.1.92.Final
netty-transport-classes-epoll-4.1.92.Final
netty-transport-native-epoll-4.1.92.Final
netty-transport-native-unix-common-4.1.92.Final
plexus-utils-3.3.0
reload4j-1.2.19
reload4j-1.2.25
rocksdbjni-7.1.2
scala-collection-compat_2.13-2.6.0
scala-collection-compat_2.13-2.10.0
scala-library-2.13.10
scala-logging_2.13-3.9.4
scala-reflect-2.13.10
scala-java8-compat_2.13-1.0.2
snakeyaml-1.30
snappy-java-1.1.8.4
swagger-annotations-2.2.0
swagger-core-2.2.0
swagger-integration-2.2.0
swagger-jaxrs2-2.2.0
swagger-models-2.2.0
snappy-java-1.1.10.0
swagger-annotations-2.2.8
zookeeper-3.6.4
zookeeper-jute-3.6.4

Expand Down Expand Up @@ -292,18 +283,21 @@ hk2-utils-2.6.1
osgi-resource-locator-1.0.3
aopalliance-repackaged-2.6.1
jakarta.inject-2.6.1
jersey-container-servlet-2.34
jersey-container-servlet-core-2.34
jersey-client-2.34
jersey-hk2-2.34
jersey-media-jaxb-2.31
jersey-client-2.39.1
jersey-common-2.39.1
jersey-container-servlet-2.39.1
jersey-container-servlet-core-2.39.1
jersey-hk2-2.39.1
jersey-server-2.39.1

---------------------------------------
CDDL 1.1 + GPLv2 with classpath exception
see: licenses/CDDL+GPL-1.1

javax.activation-api-1.2.0
javax.annotation-api-1.3.2
javax.servlet-api-3.1.0
jaxb-api-2.3.0
javax.ws.rs-api-2.1.1
jaxb-api-2.3.1
activation-1.1.1

---------------------------------------
Expand All @@ -318,12 +312,12 @@ classgraph-4.8.138, see: license/classgraph-MIT
---------------------------------------
BSD 2-Clause

zstd-jni-1.5.2-1 see: licenses/zstd-jni-BSD-2-clause
zstd-jni-1.5.5-1 see: licenses/zstd-jni-BSD-2-clause

---------------------------------------
BSD 3-Clause

jline-3.21.0, see: licenses/jline-BSD-3-clause
jline-3.22.0, see: licenses/jline-BSD-3-clause
paranamer-2.8, see: licenses/paranamer-BSD-3-clause

---------------------------------------
Expand Down
17 changes: 17 additions & 0 deletions bin/kafka-e2e-latency.sh
View file
@@ -0,0 +1,17 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.EndToEndLatency "$@"
17 changes: 17 additions & 0 deletions bin/kafka-jmx.sh
View file
@@ -0,0 +1,17 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.JmxTool "$@"
17 changes: 17 additions & 0 deletions bin/windows/kafka-cluster.bat
View file
@@ -0,0 +1,17 @@
@echo off
rem Licensed to the Apache Software Foundation (ASF) under one or more
rem contributor license agreements. See the NOTICE file distributed with
rem this work for additional information regarding copyright ownership.
rem The ASF licenses this file to You under the Apache License, Version 2.0
rem (the "License"); you may not use this file except in compliance with
rem the License. You may obtain a copy of the License at
rem
rem http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.

"%~dp0kafka-run-class.bat" org.apache.kafka.tools.ClusterTool %*
17 changes: 17 additions & 0 deletions bin/windows/kafka-e2e-latency.bat
View file
@@ -0,0 +1,17 @@
@echo off
rem Licensed to the Apache Software Foundation (ASF) under one or more
rem contributor license agreements. See the NOTICE file distributed with
rem this work for additional information regarding copyright ownership.
rem The ASF licenses this file to You under the Apache License, Version 2.0
rem (the "License"); you may not use this file except in compliance with
rem the License. You may obtain a copy of the License at
rem
rem http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.

"%~dp0kafka-run-class.bat" org.apache.kafka.tools.EndToEndLatency %*
17 changes: 17 additions & 0 deletions bin/windows/kafka-jmx.bat
View file
@@ -0,0 +1,17 @@
@echo off
rem Licensed to the Apache Software Foundation (ASF) under one or more
rem contributor license agreements. See the NOTICE file distributed with
rem this work for additional information regarding copyright ownership.
rem The ASF licenses this file to You under the Apache License, Version 2.0
rem (the "License"); you may not use this file except in compliance with
rem the License. You may obtain a copy of the License at
rem
rem http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.

"%~dp0kafka-run-class.bat" org.apache.kafka.tools.JmxTool %*
7 changes: 5 additions & 2 deletions build.gradle
View file
Expand Up @@ -167,7 +167,9 @@ allprojects {
// be explicit about the Netty dependency version instead of relying on the version set by
// ZooKeeper (potentially older and containing CVEs)
libs.nettyHandler,
libs.nettyTransportNativeEpoll
libs.nettyTransportNativeEpoll,
// be explicit about the reload4j version instead of relying on the transitive versions
libs.log4j
)
}
}
Expand Down Expand Up @@ -302,7 +304,7 @@ subprojects {
options.release = minJavaVersion
dependsOn installGitHooks
}

// We should only set this if Java version is < 9 (--release is recommended for >= 9), but the Scala plugin for IntelliJ sets
// `-target` incorrectly if this is unset
sourceCompatibility = minJavaVersion
Expand Down Expand Up @@ -593,6 +595,7 @@ subprojects {
options.charSet = 'UTF-8'
options.docEncoding = 'UTF-8'
options.encoding = 'UTF-8'
options.memberLevel = JavadocMemberLevel.PUBLIC // Document only public members/API
// Turn off doclint for now, see https://blog.joda.org/2014/02/turning-off-doclint-in-jdk-8-javadoc.html for rationale
options.addStringOption('Xdoclint:none', '-quiet')

Expand Down
8 changes: 8 additions & 0 deletions checkstyle/import-control.xml
View file
Expand Up @@ -238,6 +238,7 @@
<allow pkg="org.apache.kafka.server.config" />
<allow pkg="org.apache.kafka.server.fault" />
<allow pkg="org.apache.kafka.server.metrics" />
<allow pkg="org.apache.kafka.server.mutable" />
<allow pkg="org.apache.kafka.server.policy"/>
<allow pkg="org.apache.kafka.server.util"/>
<allow pkg="org.apache.kafka.snapshot" />
Expand Down Expand Up @@ -275,6 +276,7 @@
<allow pkg="org.apache.kafka.common.message" />
<allow pkg="org.apache.kafka.common.metadata" />
<allow pkg="org.apache.kafka.common.protocol" />
<allow pkg="org.apache.kafka.common.quota" />
<allow pkg="org.apache.kafka.common.record" />
<allow pkg="org.apache.kafka.common.resource" />
<allow pkg="org.apache.kafka.common.requests" />
Expand All @@ -297,6 +299,9 @@
<allow pkg="org.apache.kafka.metadata" />
<allow pkg="org.apache.kafka.common.internals" />
</subpackage>
<subpackage name="migration">
<allow pkg="org.apache.kafka.controller" />
</subpackage>
<subpackage name="bootstrap">
<allow pkg="org.apache.kafka.snapshot" />
</subpackage>
Expand Down Expand Up @@ -407,7 +412,10 @@
<allow pkg="org.apache.kafka.queue"/>
<allow pkg="org.apache.kafka.raft"/>
<allow pkg="org.apache.kafka.server.common" />
<allow pkg="org.apache.kafka.server.fault" />
<allow pkg="org.apache.kafka.shell"/>
<allow pkg="org.apache.kafka.image"/>
<allow pkg="org.apache.kafka.image.loader"/>
<allow pkg="org.apache.kafka.snapshot"/>
<allow pkg="org.jline"/>
<allow pkg="scala.compat"/>
Expand Down
6 changes: 3 additions & 3 deletions checkstyle/suppressions.xml
View file
Expand Up @@ -300,15 +300,15 @@

<!-- metadata -->
<suppress checks="ClassDataAbstractionCoupling"
files="(QuorumController|QuorumControllerTest|ReplicationControlManager|ReplicationControlManagerTest|ClusterControlManagerTest).java"/>
files="(QuorumController|QuorumControllerTest|ReplicationControlManager|ReplicationControlManagerTest|ClusterControlManagerTest|KRaftMigrationDriverTest).java"/>
<suppress checks="ClassFanOutComplexity"
files="(QuorumController|QuorumControllerTest|ReplicationControlManager|ReplicationControlManagerTest).java"/>
<suppress checks="(ParameterNumber|ClassDataAbstractionCoupling)"
files="(QuorumController).java"/>
<suppress checks="CyclomaticComplexity"
files="(ClientQuotasImage|KafkaEventQueue|MetadataDelta|QuorumController|ReplicationControlManager).java"/>
files="(ClientQuotasImage|KafkaEventQueue|MetadataDelta|QuorumController|ReplicationControlManager|KRaftMigrationDriver|ClusterControlManager).java"/>
<suppress checks="NPathComplexity"
files="(ClientQuotasImage|KafkaEventQueue|ReplicationControlManager|FeatureControlManager|KRaftMigrationDriver).java"/>
files="(ClientQuotasImage|KafkaEventQueue|ReplicationControlManager|FeatureControlManager|KRaftMigrationDriver|ScramControlManager|ClusterControlManager).java"/>
<suppress checks="(NPathComplexity|ClassFanOutComplexity|CyclomaticComplexity|ClassDataAbstractionCoupling|LocalVariableName|MemberName|ParameterName|MethodLength|JavaNCSS|AvoidStarImport)"
files="metadata[\\/]src[\\/](generated|generated-test)[\\/].+.java$"/>
<suppress checks="BooleanExpressionComplexity"
Expand Down
7 changes: 4 additions & 3 deletions clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
View file
Expand Up @@ -956,12 +956,13 @@ private void handleApiVersionsResponse(List<ClientResponse> responses,
}
NodeApiVersions nodeVersionInfo = new NodeApiVersions(
apiVersionsResponse.data().apiKeys(),
apiVersionsResponse.data().supportedFeatures());
apiVersionsResponse.data().supportedFeatures(),
apiVersionsResponse.data().zkMigrationReady());
apiVersions.update(node, nodeVersionInfo);
this.connectionStates.ready(node);
log.debug("Node {} has finalized features epoch: {}, finalized features: {}, supported features: {}, API versions: {}.",
log.debug("Node {} has finalized features epoch: {}, finalized features: {}, supported features: {}, ZK migration ready: {}, API versions: {}.",
node, apiVersionsResponse.data().finalizedFeaturesEpoch(), apiVersionsResponse.data().finalizedFeatures(),
apiVersionsResponse.data().supportedFeatures(), nodeVersionInfo);
apiVersionsResponse.data().supportedFeatures(), apiVersionsResponse.data().zkMigrationReady(), nodeVersionInfo);
}

/**
Expand Down
11 changes: 9 additions & 2 deletions clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java
View file
Expand Up @@ -48,6 +48,8 @@ public class NodeApiVersions {

private final Map<String, SupportedVersionRange> supportedFeatures;

private final boolean zkMigrationEnabled;

/**
* Create a NodeApiVersions object with the current ApiVersions.
*
Expand Down Expand Up @@ -76,7 +78,7 @@ public static NodeApiVersions create(Collection<ApiVersion> overrides) {
}
if (!exists) apiVersions.add(ApiVersionsResponse.toApiVersion(apiKey));
}
return new NodeApiVersions(apiVersions, Collections.emptyList());
return new NodeApiVersions(apiVersions, Collections.emptyList(), false);
}


Expand All @@ -95,7 +97,7 @@ public static NodeApiVersions create(short apiKey, short minVersion, short maxVe
.setMaxVersion(maxVersion)));
}

public NodeApiVersions(Collection<ApiVersion> nodeApiVersions, Collection<SupportedFeatureKey> nodeSupportedFeatures) {
public NodeApiVersions(Collection<ApiVersion> nodeApiVersions, Collection<SupportedFeatureKey> nodeSupportedFeatures, boolean zkMigrationEnabled) {
for (ApiVersion nodeApiVersion : nodeApiVersions) {
if (ApiKeys.hasId(nodeApiVersion.apiKey())) {
ApiKeys nodeApiKey = ApiKeys.forId(nodeApiVersion.apiKey());
Expand All @@ -112,6 +114,7 @@ public NodeApiVersions(Collection<ApiVersion> nodeApiVersions, Collection<Suppor
new SupportedVersionRange(supportedFeature.minVersion(), supportedFeature.maxVersion()));
}
this.supportedFeatures = Collections.unmodifiableMap(supportedFeaturesBuilder);
this.zkMigrationEnabled = zkMigrationEnabled;
}

/**
Expand Down Expand Up @@ -236,4 +239,8 @@ public Map<ApiKeys, ApiVersion> allSupportedApiVersions() {
public Map<String, SupportedVersionRange> supportedFeatures() {
return supportedFeatures;
}

public boolean zkMigrationEnabled() {
return zkMigrationEnabled;
}
}

0 comments on commit d14b738

Please sign in to comment.