Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/spring-batch-bigquery.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- name: Set up JDK 17
uses: actions/setup-java@v4
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: '17'
Expand Down
78 changes: 66 additions & 12 deletions spring-batch-bigquery/pom.xml
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.5.6</version>
<version>4.0.0</version>
<relativePath/>
</parent>

Expand All @@ -49,6 +49,10 @@
</developer>
</developers>

<properties>
<avro.version>1.12.1</avro.version>
</properties>

<scm>
<url>https://github.com/spring-projects/spring-batch-extensions</url>
<connection>git://github.com/spring-projects/spring-batch-extensions.git</connection>
Expand All @@ -60,7 +64,7 @@
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigquery</artifactId>
<version>2.51.0</version>
<version>2.56.0</version>
<exclusions>
<!-- Avoid version conflicts -->
<exclusion>
Expand All @@ -76,26 +80,48 @@

<!-- Optional -->
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<groupId>tools.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-csv</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>tools.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>tools.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<optional>true</optional>
</dependency>

<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
<version>1.15.2</version>
<version>1.16.0</version>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
<optional>true</optional>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.4.1</version>
<version>3.4.2</version>
<optional>true</optional>
<exclusions>
<!-- End implementation should not be forced by library -->
Expand Down Expand Up @@ -125,6 +151,10 @@
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand Down Expand Up @@ -152,6 +182,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>1.21.3</version>
<scope>test</scope>
<exclusions>
<!-- Avoid version conflicts -->
Expand All @@ -164,7 +195,7 @@
<dependency>
<groupId>org.wiremock</groupId>
<artifactId>wiremock-standalone</artifactId>
<version>3.13.1</version>
<version>3.13.2</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -201,6 +232,11 @@
<goals>
<goal>jar</goal>
</goals>
<configuration>
<excludes>
<exclude>org.springframework.batch.extensions.bigquery.common.generated.*</exclude>
</excludes>
</configuration>
</execution>
</executions>
</plugin>
Expand All @@ -222,7 +258,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.7.1</version>
<version>1.7.3</version>
<executions>
<execution>
<id>flatten</id>
Expand Down Expand Up @@ -267,17 +303,35 @@
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
<version>1.12.0</version>
<version>${avro.version}</version>
<executions>
<execution>
<id>schemas</id>
<phase>generate-sources</phase>
<phase>generate-test-sources</phase>
<goals>
<goal>schema</goal>
</goals>
<configuration>
<sourceDirectory>${project.basedir}/src/test/resources/</sourceDirectory>
<outputDirectory>${project.basedir}/target/generated-test-sources/avro/</outputDirectory>
<sourceDirectory>${project.basedir}/src/test/resources</sourceDirectory>
<outputDirectory>${project.build.directory}/generated-test-sources/avro</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<!-- Copies classes from generated-test-sources to test-classes -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.6.1</version>
<executions>
<execution>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>${project.build.directory}/generated-test-sources/avro</source>
</sources>
</configuration>
</execution>
</executions>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import com.google.cloud.bigquery.QueryJobConfiguration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.infrastructure.item.ItemReader;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.convert.converter.Converter;
import org.springframework.util.Assert;
Expand Down Expand Up @@ -64,6 +64,12 @@ public class BigQueryQueryItemReader<T> implements ItemReader<T>, InitializingBe

private Iterator<FieldValueList> iterator;

/**
* Default constructor
*/
public BigQueryQueryItemReader() {
}

/**
* BigQuery service, responsible for API calls.
* @param bigQuery BigQuery service
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ public class BigQueryQueryItemReaderBuilder<T> {

private Class<T> targetType;

/**
* Default constructor
*/
public BigQueryQueryItemReaderBuilder() {
}

/**
* BigQuery service, responsible for API calls.
* @param bigQuery BigQuery service
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,20 +35,25 @@ public final class RecordMapper<T> {

private final SimpleTypeConverter simpleConverter = new SimpleTypeConverter();

/**
* Default constructor
*/
public RecordMapper() {
}

/**
* Generates a conversion from BigQuery response to a Java record.
* @param targetType a Java record {@link Class}
* @param targetType a {@link Record}
* @return {@link Converter}
* @see org.springframework.batch.item.file.mapping.RecordFieldSetMapper
* @see org.springframework.batch.infrastructure.item.file.mapping.RecordFieldSetMapper
*/
public Converter<FieldValueList, T> generateMapper(Class<T> targetType) {
Constructor<T> constructor = BeanUtils.getResolvableConstructor(targetType);
public Converter<FieldValueList, T> generateMapper(final Class<T> targetType) {
final Constructor<T> constructor = BeanUtils.getResolvableConstructor(targetType);
Assert.isTrue(constructor.getParameterCount() > 0, "Record without fields is redundant");

String[] parameterNames = BeanUtils.getParameterNames(constructor);
Class<?>[] parameterTypes = constructor.getParameterTypes();

Object[] args = new Object[parameterNames.length];
final String[] parameterNames = BeanUtils.getParameterNames(constructor);
final Class<?>[] parameterTypes = constructor.getParameterTypes();
final Object[] args = new Object[parameterNames.length];

return source -> {
if (args[0] == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@

package org.springframework.batch.extensions.bigquery.writer;

import org.springframework.batch.item.Chunk;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.ItemWriterException;
import org.springframework.batch.infrastructure.item.Chunk;
import org.springframework.batch.infrastructure.item.ItemWriter;
import org.springframework.batch.infrastructure.item.ItemWriterException;

/**
* Unchecked {@link Exception} indicating that an error has occurred on during
Expand All @@ -35,15 +35,15 @@ public class BigQueryItemWriterException extends ItemWriterException {
* @param message the message for this {@link Exception}
* @param cause the other {@link Exception}
*/
public BigQueryItemWriterException(String message, Throwable cause) {
public BigQueryItemWriterException(final String message, final Throwable cause) {
super(message, cause);
}

/**
* Create a new {@link BigQueryItemWriterException} based on a message.
* @param message the message for this {@link Exception}
*/
public BigQueryItemWriterException(String message) {
public BigQueryItemWriterException(final String message) {
super(message);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,20 @@

package org.springframework.batch.extensions.bigquery.writer.loadjob;

import com.google.cloud.bigquery.*;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableDataWriteChannel;
import com.google.cloud.bigquery.TableDefinition;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.WriteChannelConfiguration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.extensions.bigquery.writer.BigQueryItemWriterException;
import org.springframework.batch.item.Chunk;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.infrastructure.item.Chunk;
import org.springframework.batch.infrastructure.item.ItemWriter;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;

Expand All @@ -43,6 +51,12 @@
*/
public abstract class BigQueryLoadJobBaseItemWriter<T> implements ItemWriter<T>, InitializingBean {

/**
* Default constructor
*/
protected BigQueryLoadJobBaseItemWriter() {
}

/** Logger that can be reused */
protected final Log logger = LogFactory.getLog(getClass());

Expand Down
Loading