Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

update confluent dependency version to 7.6.0 #30638

Merged
merged 8 commits into from
Mar 19, 2024
Merged

update confluent dependency version to 7.6.0 #30638

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e95e715
update confluent dependency version to 7.6.0
tilgalas Mar 8, 2024
18ac188
make avroSchema field non-transient as the Schema class is now Serial…
tilgalas Mar 14, 2024
b51a96f
remove now unused org.codehaus.jackson package from GcpApiSurfaceTest
tilgalas Mar 14, 2024
4e91c0d
update expected exception class in PubsubClientTest
tilgalas Mar 14, 2024
824a076
fix AvroGenericRecordToStorageApiProtoTest
tilgalas Mar 15, 2024
fd0d50d
move org.tukaani:xz:1.9 dependency to testImplementation
tilgalas Mar 15, 2024
6839a3d
bring back the tukaani dep to the avroVersions configurations
tilgalas Mar 15, 2024
11dd82f
trigger the beam_PostCommitJava_Avro_Versions workflow
tilgalas Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion buildSrc/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ dependencies {
implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.14")

runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation
runtimeOnly("com.github.davidmc24.gradle-avro-plugin:gradle-avro-plugin:0.16.0") // Enable Avro code generation
Abacn marked this conversation as resolved.
Show resolved Hide resolved
runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1") // Enable Avro code generation
runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin
runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree
runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") // Enable shading Java dependencies
Expand Down
6 changes: 3 additions & 3 deletions buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -664,8 +664,8 @@ class BeamModulePlugin implements Plugin<Project> {
antlr_runtime : "org.antlr:antlr4-runtime:4.7",
args4j : "args4j:args4j:2.33",
auto_value_annotations : "com.google.auto.value:auto-value-annotations:$autovalue_version",
avro : "org.apache.avro:avro:1.8.2",
avro_tests : "org.apache.avro:avro:1.8.2:tests",
avro : "org.apache.avro:avro:1.11.3",
avro_tests : "org.apache.avro:avro:1.11.3:tests",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I remember there was issue upgrading avro version and that was why it sticked to 1.8.2 for a long time. Might be good to ask in devlist about this proposal

aws_java_sdk_cloudwatch : "com.amazonaws:aws-java-sdk-cloudwatch:$aws_java_sdk_version",
aws_java_sdk_core : "com.amazonaws:aws-java-sdk-core:$aws_java_sdk_version",
aws_java_sdk_dynamodb : "com.amazonaws:aws-java-sdk-dynamodb:$aws_java_sdk_version",
Expand Down Expand Up @@ -2448,7 +2448,7 @@ class BeamModulePlugin implements Plugin<Project> {
// TODO: Decide whether this should be inlined into the one project that relies on it
// or be left here.
project.ext.applyAvroNature = {
project.apply plugin: "com.commercehub.gradle.plugin.avro"
project.apply plugin: "com.github.davidmc24.gradle.plugin.avro"

// add dependency BeamModulePlugin defined custom tasks
// they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc)
Expand Down
2 changes: 1 addition & 1 deletion sdks/java/extensions/avro/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ dependencies {
}
implementation library.java.error_prone_annotations
implementation library.java.avro
implementation "org.tukaani:xz:1.9" // marked as optional in avro
Abacn marked this conversation as resolved.
Show resolved Hide resolved
implementation library.java.joda_time
testImplementation(project(path: ":sdks:java:core", configuration: "shadowTest")) {
// Exclude Avro dependencies from "core" since Avro support moved to this extension
Expand All @@ -82,7 +83,6 @@ dependencies {
"avroVersion$k" library.java.junit
"avroVersion$k" project(path: ":runners:direct-java", configuration: "shadow")
"avroVersion$k" library.java.slf4j_jdk14
"avroVersion$k" "org.tukaani:xz:1.9" // marked as optional in avro
"avroVersion$k" library.java.zstd_jni // marked as optional in avro
"avroVersion${k}Generate" "org.apache.avro:avro-tools:$v"
}
Expand Down
2 changes: 1 addition & 1 deletion ...oud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ public void testConvertGenericRecordToTableRow() throws Exception {
}
// After a Field is added to a Schema, it is assigned a position, so we can't simply reuse
// the existing Field.
avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultValue()));
avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultVal()));
}
Schema avroSchema = Schema.createRecord(avroFields);

Expand Down
9 changes: 6 additions & 3 deletions sdks/java/io/kafka/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@ provideIntegrationTestingDependencies()
enableJavaPerformanceTesting()

description = "Apache Beam :: SDKs :: Java :: IO :: Kafka"
ext.summary = "Library to read Kafka topics."
ext {
summary = "Library to read Kafka topics."
confluentVersion = "7.6.0"
}

def kafkaVersions = [
'01103': "0.11.0.3",
Expand Down Expand Up @@ -68,14 +71,14 @@ dependencies {
implementation library.java.jackson_annotations
implementation library.java.jackson_databind
implementation "org.springframework:spring-expression:5.3.27"
implementation ("io.confluent:kafka-avro-serializer:5.3.2") {
implementation ("io.confluent:kafka-avro-serializer:${confluentVersion}") {
// zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with current
// "spotbugs-annotations:3.1.12" used in Beam. Not required.
exclude group: "org.apache.zookeeper", module: "zookeeper"
// "kafka-clients" has to be provided since user can use its own version.
exclude group: "org.apache.kafka", module: "kafka-clients"
}
implementation ("io.confluent:kafka-schema-registry-client:5.3.2") {
implementation ("io.confluent:kafka-schema-registry-client:${confluentVersion}") {
// It depends on "spotbugs-annotations:3.1.9" which clashes with current
// "spotbugs-annotations:3.1.12" used in Beam. Not required.
exclude group: "org.apache.zookeeper", module: "zookeeper"
Expand Down