Skip to content
Permalink
Browse files

Spring batch configuration moved from XML to Java

  • Loading branch information...
cezarykluczynski committed Dec 11, 2016
1 parent 95c510a commit 0e290e786e59a0694059a2209f6861d596f40d32
Showing with 938 additions and 129 deletions.
  1. +1 −2 .gitignore
  2. +2 −1 ...m/cezarykluczynski/stapi/etl/character/creation/configuration/CharacterCreationConfiguration.java
  3. +0 −1 ...src/main/java/com/cezarykluczynski/stapi/etl/character/creation/processor/CharacterProcessor.java
  4. +17 −9 etl/src/main/java/com/cezarykluczynski/stapi/etl/common/service/JobCompletenessDecider.java
  5. +2 −1 etl/src/main/java/com/cezarykluczynski/stapi/etl/configuration/EtlConfiguration.java
  6. +123 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/configuration/job/EtlJobConfiguration.java
  7. +60 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/configuration/job/JobBuilder.java
  8. +98 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/configuration/job/StepConfigurationValidator.java
  9. +18 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/configuration/job/StepDependency.java
  10. +14 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/configuration/job/properties/StepProperties.java
  11. +36 −0 ...ava/com/cezarykluczynski/stapi/etl/configuration/job/properties/StepToStepPropertiesProvider.java
  12. +22 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/configuration/job/properties/StepsProperties.java
  13. +2 −1 ...a/com/cezarykluczynski/stapi/etl/episode/creation/configuration/EpisodeCreationConfiguration.java
  14. +2 −1 ...m/cezarykluczynski/stapi/etl/performer/creation/configuration/PerformerCreationConfiguration.java
  15. +2 −1 ...ava/com/cezarykluczynski/stapi/etl/series/creation/configuration/SeriesCreationConfiguration.java
  16. +2 −1 .../java/com/cezarykluczynski/stapi/etl/staff/creation/configuration/StaffCreationConfiguration.java
  17. +7 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/util/constant/JobName.java
  18. +11 −0 etl/src/main/java/com/cezarykluczynski/stapi/etl/util/constant/StepName.java
  19. +0 −78 etl/src/main/resources/spring/batch/jobs/create.xml
  20. +3 −2 ...rykluczynski/stapi/etl/character/creation/configuration/CharacterCreationConfigurationTest.groovy
  21. +35 −6 etl/src/test/groovy/com/cezarykluczynski/stapi/etl/common/service/JobCompletenessDeciderTest.groovy
  22. +265 −0 etl/src/test/groovy/com/cezarykluczynski/stapi/etl/configuration/job/EtlJobConfigurationTest.groovy
  23. +88 −0 etl/src/test/groovy/com/cezarykluczynski/stapi/etl/configuration/job/JobBuilderTest.groovy
  24. +84 −0 ...est/groovy/com/cezarykluczynski/stapi/etl/configuration/job/StepConfigurationValidatorTest.groovy
  25. +3 −2 ...cezarykluczynski/stapi/etl/episode/creation/configuration/EpisodeCreationConfigurationTest.groovy
  26. +3 −2 ...rykluczynski/stapi/etl/performer/creation/configuration/PerformerCreationConfigurationTest.groovy
  27. +3 −2 ...m/cezarykluczynski/stapi/etl/series/creation/configuration/SeriesCreationConfigurationTest.groovy
  28. +3 −2 ...com/cezarykluczynski/stapi/etl/staff/creation/configuration/StaffCreationConfigurationTest.groovy
  29. +15 −0 server/src/main/resources/etl/application-etl.properties
  30. +1 −1 server/src/test/groovy/com/cezarykluczynski/stapi/server/StaticJobCompletenessDecider.groovy
  31. +2 −2 .../com/cezarykluczynski/stapi/server/character/endpoint/CharacterRestEndpointIntegrationTest.groovy
  32. +2 −2 .../com/cezarykluczynski/stapi/server/character/endpoint/CharacterSoapEndpointIntegrationTest.groovy
  33. +2 −2 .../com/cezarykluczynski/stapi/server/performer/endpoint/PerformerRestEndpointIntegrationTest.groovy
  34. +2 −2 .../com/cezarykluczynski/stapi/server/performer/endpoint/PerformerSoapEndpointIntegrationTest.groovy
  35. +2 −2 ...groovy/com/cezarykluczynski/stapi/server/series/endpoint/SeriesRestEndpointIntegrationTest.groovy
  36. +2 −2 ...groovy/com/cezarykluczynski/stapi/server/series/endpoint/SeriesSoapEndpointIntegrationTest.groovy
  37. +2 −2 ...t/groovy/com/cezarykluczynski/stapi/server/staff/endpoint/StaffRestEndpointIntegrationTest.groovy
  38. +2 −2 ...t/groovy/com/cezarykluczynski/stapi/server/staff/endpoint/StaffSoapEndpointIntegrationTest.groovy
@@ -5,6 +5,5 @@ server/build/
model/build/
client/build/
logs/
server/src/main/resources/application-db.properties
server/src/main/resources/application-source.properties
server/src/main/resources/application-*.properties
model/**/*_.java
@@ -3,6 +3,7 @@
import com.cezarykluczynski.stapi.etl.character.creation.processor.CharacterReader;
import com.cezarykluczynski.stapi.etl.common.service.JobCompletenessDecider;
import com.cezarykluczynski.stapi.etl.util.constant.CategoryName;
import com.cezarykluczynski.stapi.etl.util.constant.StepName;
import com.cezarykluczynski.stapi.sources.mediawiki.api.CategoryApi;
import com.cezarykluczynski.stapi.sources.mediawiki.api.enums.MediaWikiSource;
import com.cezarykluczynski.stapi.sources.mediawiki.dto.PageHeader;
@@ -36,7 +37,7 @@
public CharacterReader characterReader() {
List<PageHeader> characters = Lists.newArrayList();

if (!jobCompletenessDecider.isStepComplete(JobCompletenessDecider.STEP_004_CREATE_CHARACTERS)) {
if (!jobCompletenessDecider.isStepComplete(StepName.CREATE_CHARACTERS)) {
characters.addAll(categoryApi.getPagesIncludingSubcategories(CategoryName.INDIVIDUALS, MediaWikiSource.MEMORY_ALPHA_EN));
characters.addAll(categoryApi.getPagesIncludingSubcategories(CategoryName.MILITARY_PERSONNEL, MediaWikiSource.MEMORY_ALPHA_EN));
characters.addAll(categoryApi.getPagesIncludingSubcategories(CategoryName.Q_CONTINUUM, MediaWikiSource.MEMORY_ALPHA_EN));
@@ -22,4 +22,3 @@ public CharacterProcessor(PageHeaderProcessor pageHeaderProcessor,
}

}

@@ -1,5 +1,8 @@
package com.cezarykluczynski.stapi.etl.common.service;

import com.cezarykluczynski.stapi.etl.configuration.job.properties.StepProperties;
import com.cezarykluczynski.stapi.etl.configuration.job.properties.StepToStepPropertiesProvider;
import com.cezarykluczynski.stapi.etl.util.constant.JobName;
import com.google.common.collect.Lists;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobExecution;
@@ -11,25 +14,30 @@

import javax.inject.Inject;
import java.util.Collection;
import java.util.Map;

@Service
public class JobCompletenessDecider {

public static final String JOB_CREATE = "CREATE";
public static final String STEP_001_CREATE_SERIES = "STEP_001_CREATE_SERIES";
public static final String STEP_002_CREATE_PERFORMERS = "STEP_002_CREATE_PERFORMERS";
public static final String STEP_003_CREATE_STAFF = "STEP_003_CREATE_STAFF";
public static final String STEP_004_CREATE_CHARACTERS = "STEP_004_CREATE_CHARACTERS";
public static final String STEP_005_CREATE_EPISODES= "STEP_005_CREATE_EPISODES";

private JobRepository jobRepository;

private StepToStepPropertiesProvider stepToStepPropertiesProvider;

@Inject
public JobCompletenessDecider(JobRepository jobRepository) {
public JobCompletenessDecider(JobRepository jobRepository,
StepToStepPropertiesProvider stepToStepPropertiesProvider) {
this.jobRepository = jobRepository;
this.stepToStepPropertiesProvider = stepToStepPropertiesProvider;
}

public boolean isStepComplete(String stepName) {
Map<String, StepProperties> stepPropertiesMap = stepToStepPropertiesProvider.provide();
StepProperties stepProperties = stepPropertiesMap.get(stepName);

if (stepProperties != null && !stepProperties.isEnabled()) {
return true;
}

Collection<StepExecution> stepExecutions = getAllStepExecutions();
return stepExecutions.stream().anyMatch(stepExecution ->
stepExecution.getStepName().equals(stepName) && BatchStatus.COMPLETED.equals(stepExecution.getStatus()));
@@ -40,7 +48,7 @@ public boolean isStepComplete(String stepName) {

try {
jobExecution = jobRepository
.getLastJobExecution(JOB_CREATE, new JobParameters());
.getLastJobExecution(JobName.JOB_CREATE, new JobParameters());
} catch (BadSqlGrammarException e) {
return Lists.newArrayList();
}
@@ -1,5 +1,6 @@
package com.cezarykluczynski.stapi.etl.configuration;

import com.cezarykluczynski.stapi.etl.configuration.job.EtlJobConfiguration;
import com.cezarykluczynski.stapi.util.constant.SpringProfile;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.context.annotation.*;
@@ -9,7 +10,7 @@
@Profile(SpringProfile.ETL)
@Configuration
@EnableBatchProcessing
@ImportResource(locations = {"classpath:spring/batch/jobs/create.xml"})
@Import(EtlJobConfiguration.class)
@ComponentScan({
"com.cezarykluczynski.stapi.etl",
"com.cezarykluczynski.stapi.sources"
@@ -0,0 +1,123 @@
package com.cezarykluczynski.stapi.etl.configuration.job;

import com.cezarykluczynski.stapi.etl.character.creation.processor.CharacterProcessor;
import com.cezarykluczynski.stapi.etl.character.creation.processor.CharacterReader;
import com.cezarykluczynski.stapi.etl.character.creation.processor.CharacterWriter;
import com.cezarykluczynski.stapi.etl.common.listener.CommonStepExecutionListener;
import com.cezarykluczynski.stapi.etl.configuration.job.properties.StepsProperties;
import com.cezarykluczynski.stapi.etl.episode.creation.processor.EpisodeProcessor;
import com.cezarykluczynski.stapi.etl.episode.creation.processor.EpisodeReader;
import com.cezarykluczynski.stapi.etl.episode.creation.processor.EpisodeWriter;
import com.cezarykluczynski.stapi.etl.performer.creation.processor.PerformerProcessor;
import com.cezarykluczynski.stapi.etl.performer.creation.processor.PerformerReader;
import com.cezarykluczynski.stapi.etl.performer.creation.processor.PerformerWriter;
import com.cezarykluczynski.stapi.etl.series.creation.processor.SeriesProcessor;
import com.cezarykluczynski.stapi.etl.series.creation.processor.SeriesReader;
import com.cezarykluczynski.stapi.etl.series.creation.processor.SeriesWriter;
import com.cezarykluczynski.stapi.etl.staff.creation.processor.StaffProcessor;
import com.cezarykluczynski.stapi.etl.staff.creation.processor.StaffReader;
import com.cezarykluczynski.stapi.etl.staff.creation.processor.StaffWriter;
import com.cezarykluczynski.stapi.etl.util.constant.StepName;
import com.cezarykluczynski.stapi.model.character.entity.Character;
import com.cezarykluczynski.stapi.model.episode.entity.Episode;
import com.cezarykluczynski.stapi.model.performer.entity.Performer;
import com.cezarykluczynski.stapi.model.series.entity.Series;
import com.cezarykluczynski.stapi.model.staff.entity.Staff;
import com.cezarykluczynski.stapi.sources.mediawiki.dto.PageHeader;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import javax.inject.Inject;

@Configuration
@EnableConfigurationProperties(value = StepsProperties.class)
public class EtlJobConfiguration {

@Inject
private ApplicationContext applicationContext;

@Inject
private JobBuilder jobBuilder;

@Inject
private StepBuilderFactory stepBuilderFactory;

@Inject
private StepsProperties stepsProperties;

@Bean
public Job jobCreate() {
return jobBuilder.build();
}

@Bean(name = StepName.CREATE_SERIES)
public Step stepCreateSeries() {
return stepBuilderFactory.get(StepName.CREATE_SERIES)
.<PageHeader, Series> chunk(stepsProperties.getCreateSeries().getCommitInterval())
.reader(applicationContext.getBean(SeriesReader.class))
.processor(applicationContext.getBean(SeriesProcessor.class))
.writer(applicationContext.getBean(SeriesWriter.class))
.listener(applicationContext.getBean(CommonStepExecutionListener.class))
.startLimit(1)
.allowStartIfComplete(false)
.build();
}

@Bean(name = StepName.CREATE_PERFORMERS)
public Step stepCreatePerformers() {
return stepBuilderFactory.get(StepName.CREATE_PERFORMERS)
.<PageHeader, Performer> chunk(stepsProperties.getCreatePerformers().getCommitInterval())
.reader(applicationContext.getBean(PerformerReader.class))
.processor(applicationContext.getBean(PerformerProcessor.class))
.writer(applicationContext.getBean(PerformerWriter.class))
.listener(applicationContext.getBean(CommonStepExecutionListener.class))
.startLimit(1)
.allowStartIfComplete(false)
.build();
}

@Bean(name = StepName.CREATE_STAFF)
public Step stepCreateStaff() {
return stepBuilderFactory.get(StepName.CREATE_STAFF)
.<PageHeader, Staff> chunk(stepsProperties.getCreateStaff().getCommitInterval())
.reader(applicationContext.getBean(StaffReader.class))
.processor(applicationContext.getBean(StaffProcessor.class))
.writer(applicationContext.getBean(StaffWriter.class))
.listener(applicationContext.getBean(CommonStepExecutionListener.class))
.startLimit(1)
.allowStartIfComplete(false)
.build();
}

@Bean(name = StepName.CREATE_CHARACTERS)
public Step stepCreateCharacters() {
return stepBuilderFactory.get(StepName.CREATE_CHARACTERS)
.<PageHeader, Character> chunk(stepsProperties.getCreateCharacters().getCommitInterval())
.reader(applicationContext.getBean(CharacterReader.class))
.processor(applicationContext.getBean(CharacterProcessor.class))
.writer(applicationContext.getBean(CharacterWriter.class))
.listener(applicationContext.getBean(CommonStepExecutionListener.class))
.startLimit(1)
.allowStartIfComplete(false)
.build();
}

@Bean(name = StepName.CREATE_EPISODES)
public Step stepCreateEpisodes() {
return stepBuilderFactory.get(StepName.CREATE_EPISODES)
.<PageHeader, Episode> chunk(stepsProperties.getCreateEpisodes().getCommitInterval())
.reader(applicationContext.getBean(EpisodeReader.class))
.processor(applicationContext.getBean(EpisodeProcessor.class))
.writer(applicationContext.getBean(EpisodeWriter.class))
.listener(applicationContext.getBean(CommonStepExecutionListener.class))
.startLimit(1)
.allowStartIfComplete(false)
.build();
}

}
@@ -0,0 +1,60 @@
package com.cezarykluczynski.stapi.etl.configuration.job;

import com.cezarykluczynski.stapi.etl.util.constant.JobName;
import com.cezarykluczynski.stapi.etl.util.constant.StepName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.job.builder.FlowBuilder;
import org.springframework.batch.core.job.builder.SimpleJobBuilder;
import org.springframework.batch.core.job.flow.Flow;
import org.springframework.context.ApplicationContext;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;

import javax.inject.Inject;

@Service
@Slf4j
public class JobBuilder {

private ApplicationContext applicationContext;

private JobBuilderFactory jobBuilderFactory;

private StepConfigurationValidator stepConfigurationValidator;

@Inject
public JobBuilder(ApplicationContext applicationContext, JobBuilderFactory jobBuilderFactory,
StepConfigurationValidator stepConfigurationValidator) {
this.applicationContext = applicationContext;
this.jobBuilderFactory = jobBuilderFactory;
this.stepConfigurationValidator = stepConfigurationValidator;
}

public synchronized Job build() {
stepConfigurationValidator.validate();

org.springframework.batch.core.job.builder.JobBuilder jobBuilder = jobBuilderFactory.get(JobName.JOB_CREATE);
SimpleJobBuilder simpleJobBuilder = new SimpleJobBuilder(jobBuilder);

Flow flow1 = new FlowBuilder<Flow>("flow1")
.from(applicationContext.getBean(StepName.CREATE_SERIES, Step.class))
.end();

Flow flow2 = new FlowBuilder<Flow>("flow2")
.from(applicationContext.getBean(StepName.CREATE_PERFORMERS, Step.class))
.next(applicationContext.getBean(StepName.CREATE_STAFF, Step.class))
.next(applicationContext.getBean(StepName.CREATE_CHARACTERS, Step.class))
.next(applicationContext.getBean(StepName.CREATE_EPISODES, Step.class))
.end();

return simpleJobBuilder
.split(applicationContext.getBean(TaskExecutor.class))
.add(flow1, flow2)
.end()
.build();
}
}

@@ -0,0 +1,98 @@
package com.cezarykluczynski.stapi.etl.configuration.job;

import com.cezarykluczynski.stapi.etl.configuration.job.properties.StepProperties;
import com.cezarykluczynski.stapi.etl.configuration.job.properties.StepsProperties;
import com.cezarykluczynski.stapi.etl.util.constant.StepName;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.springframework.batch.core.job.builder.JobBuilderException;
import org.springframework.stereotype.Service;

import javax.inject.Inject;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;

@Service
public class StepConfigurationValidator {

private static final long NUMBER_OF_STEPS = 5;

private StepsProperties stepsProperties;

private List<StepProperties> stepPropertiesList;

private Map<String, StepProperties> stepPropertiesMap;

@Inject
public StepConfigurationValidator(StepsProperties stepsProperties) {
this.stepsProperties = stepsProperties;
}

public void validate() {
addAllToList();
addAllToMap();
validateNumberOfSteps();
validateOrder();
}

private void addAllToList() {
stepPropertiesList = Lists.newArrayList();
stepPropertiesList.add(stepsProperties.getCreateSeries());
stepPropertiesList.add(stepsProperties.getCreatePerformers());
stepPropertiesList.add(stepsProperties.getCreateStaff());
stepPropertiesList.add(stepsProperties.getCreateCharacters());
stepPropertiesList.add(stepsProperties.getCreateEpisodes());
stepPropertiesList = stepPropertiesList.stream()
.filter(Objects::nonNull)
.collect(Collectors.toList());
}

private void addAllToMap() {
stepPropertiesMap = Maps.newHashMap();
stepPropertiesMap.put(StepName.CREATE_SERIES, stepsProperties.getCreateSeries());
stepPropertiesMap.put(StepName.CREATE_PERFORMERS, stepsProperties.getCreatePerformers());
stepPropertiesMap.put(StepName.CREATE_STAFF, stepsProperties.getCreateStaff());
stepPropertiesMap.put(StepName.CREATE_CHARACTERS, stepsProperties.getCreateCharacters());
stepPropertiesMap.put(StepName.CREATE_EPISODES, stepsProperties.getCreateEpisodes());
}

private void validateNumberOfSteps() {
long stepCount = stepPropertiesList.stream()
.filter(Objects::nonNull)
.count();

if (stepCount != NUMBER_OF_STEPS) {
doThrow(String.format("Number of configured steps is %s, but %s steps found", NUMBER_OF_STEPS, stepCount));
}
}

private void validateOrder() {
Map<String, Integer> stepOrder = Maps.newHashMap();
Set<Map.Entry<String, StepProperties>> stepEntrySet = stepPropertiesMap.entrySet();

stepEntrySet.forEach(stepPropertiesEntry ->
stepOrder.put(stepPropertiesEntry.getKey(), stepPropertiesEntry.getValue().getOrder()));

stepEntrySet.forEach(stepPropertiesEntry -> {
String rightStepName = stepPropertiesEntry.getKey();
Integer rightStepOrder = stepPropertiesEntry.getValue().getOrder();
stepOrder.entrySet().forEach(stringIntegerEntry -> {
String leftStepName = stringIntegerEntry.getKey();
Integer leftStepOrder = stringIntegerEntry.getValue();
if (!rightStepName.equals(leftStepName) && rightStepOrder.equals(leftStepOrder)) {
doThrow(String.format("Step %s has order %s, but this order was already given to step %s",
rightStepName, rightStepOrder, leftStepName));
}
});
});
}

private void doThrow(String message) {
throw new JobBuilderException(new RuntimeException(message));
}


}
Oops, something went wrong.

0 comments on commit 0e290e7

Please sign in to comment.
You can’t perform that action at this time.