-
Notifications
You must be signed in to change notification settings - Fork 32
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added batch processing for importing comics [#852]
* Added the comixed-batch submodule. * Removed the process comics task and encoder. * Changed SonarCloud to exclude classes named "*Configuration" * Renamed ComicStateMachineConfig => ComicStateMachineConfiguration * Changed the comic state machine to add more granular processing steps.
- Loading branch information
Showing
75 changed files
with
2,798 additions
and
922 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
<?xml version="1.0"?> | ||
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" | ||
xmlns="http://maven.apache.org/POM/4.0.0" | ||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> | ||
<modelVersion>4.0.0</modelVersion> | ||
<parent> | ||
<groupId>org.comixedproject</groupId> | ||
<artifactId>comixed</artifactId> | ||
<version>0.10.0-SNAPSHOT</version> | ||
<relativePath>..</relativePath> | ||
</parent> | ||
<groupId>org.comixedproject</groupId> | ||
<artifactId>comixed-batch</artifactId> | ||
<version>0.10.0-SNAPSHOT</version> | ||
<name>comixed-batch</name> | ||
<url>http://maven.apache.org</url> | ||
<dependencies> | ||
<dependency> | ||
<groupId>org.springframework.boot</groupId> | ||
<artifactId>spring-boot-starter-batch</artifactId> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.springframework.batch</groupId> | ||
<artifactId>spring-batch-test</artifactId> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.comixedproject</groupId> | ||
<artifactId>comixed-model</artifactId> | ||
<version>0.10.0-SNAPSHOT</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.comixedproject</groupId> | ||
<artifactId>comixed-adaptors</artifactId> | ||
<version>0.10.0-SNAPSHOT</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.comixedproject</groupId> | ||
<artifactId>comixed-services</artifactId> | ||
<version>0.10.0-SNAPSHOT</version> | ||
</dependency> | ||
</dependencies> | ||
</project> |
156 changes: 156 additions & 0 deletions
156
...d-batch/src/main/java/org/comixedproject/batch/comicbooks/ProcessComicsConfiguration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,156 @@ | ||
/* | ||
* ComiXed - A digital comic book library management application. | ||
* Copyright (C) 2021, The ComiXed Project | ||
* | ||
* This program is free software: you can redistribute it and/or modify | ||
* it under the terms of the GNU General Public License as published by | ||
* the Free Software Foundation, either version 3 of the License, or | ||
* (at your option) any later version. | ||
* | ||
* This program is distributed in the hope that it will be useful, | ||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
* GNU General Public License for more details. | ||
* | ||
* You should have received a copy of the GNU General Public License | ||
* along with this program. If not, see <http://www.gnu.org/licenses> | ||
*/ | ||
|
||
package org.comixedproject.batch.comicbooks; | ||
|
||
import lombok.extern.log4j.Log4j2; | ||
import org.comixedproject.batch.comicbooks.processors.ContentsProcessedProcessor; | ||
import org.comixedproject.batch.comicbooks.processors.LoadFileContentsProcessor; | ||
import org.comixedproject.batch.comicbooks.processors.LoadFileDetailsProcessor; | ||
import org.comixedproject.batch.comicbooks.processors.MarkBlockedPagesProcessor; | ||
import org.comixedproject.batch.comicbooks.readers.ContentsProcessedReader; | ||
import org.comixedproject.batch.comicbooks.readers.LoadFileContentsReader; | ||
import org.comixedproject.batch.comicbooks.readers.LoadFileDetailsReader; | ||
import org.comixedproject.batch.comicbooks.readers.MarkBlockedPagesReader; | ||
import org.comixedproject.batch.comicbooks.writers.ContentsProcessedWriter; | ||
import org.comixedproject.batch.comicbooks.writers.LoadFileContentsWriter; | ||
import org.comixedproject.batch.comicbooks.writers.LoadFileDetailsWriter; | ||
import org.comixedproject.batch.comicbooks.writers.MarkBlockedPagesWriter; | ||
import org.comixedproject.model.comic.Comic; | ||
import org.springframework.batch.core.Job; | ||
import org.springframework.batch.core.JobParametersBuilder; | ||
import org.springframework.batch.core.JobParametersInvalidException; | ||
import org.springframework.batch.core.Step; | ||
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; | ||
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; | ||
import org.springframework.batch.core.launch.JobLauncher; | ||
import org.springframework.batch.core.launch.support.RunIdIncrementer; | ||
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; | ||
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; | ||
import org.springframework.batch.core.repository.JobRestartException; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.beans.factory.annotation.Value; | ||
import org.springframework.context.annotation.Bean; | ||
import org.springframework.context.annotation.Configuration; | ||
import org.springframework.scheduling.annotation.Scheduled; | ||
|
||
/** | ||
* <code>ProcessComicsConfiguration</code> defines the batch process for importing comics. | ||
* | ||
* @author Darryl L. Pierce | ||
*/ | ||
@Configuration | ||
@Log4j2 | ||
public class ProcessComicsConfiguration { | ||
private static final String KEY_STARTED = "job.started"; | ||
|
||
@Autowired public JobBuilderFactory jobBuilderFactory; | ||
@Autowired public StepBuilderFactory stepBuilderFactory; | ||
@Autowired private JobLauncher jobLauncher; | ||
|
||
@Autowired private LoadFileContentsReader loadFileContentsReader; | ||
@Autowired private LoadFileContentsProcessor loadFileContentsProcessor; | ||
@Autowired private LoadFileContentsWriter loadFileContentsWriter; | ||
@Autowired private MarkBlockedPagesReader markBlockedPagesReader; | ||
@Autowired private MarkBlockedPagesProcessor markBlockedPagesProcessor; | ||
@Autowired private MarkBlockedPagesWriter markBlockedPagesWriter; | ||
@Autowired private LoadFileDetailsReader loadFileDetailsReader; | ||
@Autowired private LoadFileDetailsProcessor loadFileDetailsProcessor; | ||
@Autowired private LoadFileDetailsWriter loadFileDetailsWriter; | ||
@Autowired private ContentsProcessedReader contentsProcessedReader; | ||
@Autowired private ContentsProcessedProcessor contentsProcessedProcessor; | ||
@Autowired private ContentsProcessedWriter contentsProcessedWriter; | ||
|
||
@Value("${batch.chunk-size}") | ||
private int batchChunkSize = 10; | ||
|
||
@Bean | ||
public Job importComicsJob() { | ||
return this.jobBuilderFactory | ||
.get("importComicsJob") | ||
.incrementer(new RunIdIncrementer()) | ||
.start(loadFileContentsStep()) | ||
.next(markBlockedPagesStep()) | ||
.next(loadFileDetailsStep()) | ||
.next(contentsProcessedStep()) | ||
.build(); | ||
} | ||
|
||
@Bean | ||
public Step loadFileContentsStep() { | ||
return this.stepBuilderFactory | ||
.get("loadFileContentsStep") | ||
.<Comic, Comic>chunk(this.batchChunkSize) | ||
.reader(loadFileContentsReader) | ||
.processor(loadFileContentsProcessor) | ||
.writer(loadFileContentsWriter) | ||
.build(); | ||
} | ||
|
||
@Bean | ||
public Step markBlockedPagesStep() { | ||
return this.stepBuilderFactory | ||
.get("markBlockedPagesStep") | ||
.<Comic, Comic>chunk(this.batchChunkSize) | ||
.reader(markBlockedPagesReader) | ||
.processor(markBlockedPagesProcessor) | ||
.writer(markBlockedPagesWriter) | ||
.build(); | ||
} | ||
|
||
@Bean | ||
public Step loadFileDetailsStep() { | ||
return this.stepBuilderFactory | ||
.get("loadFileDetailsStep") | ||
.<Comic, Comic>chunk(this.batchChunkSize) | ||
.reader(loadFileDetailsReader) | ||
.processor(loadFileDetailsProcessor) | ||
.writer(loadFileDetailsWriter) | ||
.build(); | ||
} | ||
|
||
@Bean | ||
public Step contentsProcessedStep() { | ||
return this.stepBuilderFactory | ||
.get("contentsProcessedStep") | ||
.<Comic, Comic>chunk(this.batchChunkSize) | ||
.reader(contentsProcessedReader) | ||
.processor(contentsProcessedProcessor) | ||
.writer(contentsProcessedWriter) | ||
.build(); | ||
} | ||
|
||
/** | ||
* Runs the comic processing job. | ||
* | ||
* @throws JobInstanceAlreadyCompleteException if an error occurs | ||
* @throws JobExecutionAlreadyRunningException if an error occurs | ||
* @throws JobParametersInvalidException if an error occurs | ||
* @throws JobRestartException if an error occurs | ||
*/ | ||
@Scheduled(fixedDelay = 1000) | ||
public void performJob() | ||
throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, | ||
JobParametersInvalidException, JobRestartException { | ||
this.jobLauncher.run( | ||
importComicsJob(), | ||
new JobParametersBuilder() | ||
.addLong(KEY_STARTED, System.currentTimeMillis()) | ||
.toJobParameters()); | ||
} | ||
} |
40 changes: 40 additions & 0 deletions
40
.../main/java/org/comixedproject/batch/comicbooks/processors/ContentsProcessedProcessor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
/* | ||
* ComiXed - A digital comic book library management application. | ||
* Copyright (C) 2021, The ComiXed Project | ||
* | ||
* This program is free software: you can redistribute it and/or modify | ||
* it under the terms of the GNU General Public License as published by | ||
* the Free Software Foundation, either version 3 of the License, or | ||
* (at your option) any later version. | ||
* | ||
* This program is distributed in the hope that it will be useful, | ||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
* GNU General Public License for more details. | ||
* | ||
* You should have received a copy of the GNU General Public License | ||
* along with this program. If not, see <http://www.gnu.org/licenses> | ||
*/ | ||
|
||
package org.comixedproject.batch.comicbooks.processors; | ||
|
||
import lombok.extern.log4j.Log4j2; | ||
import org.comixedproject.model.comic.Comic; | ||
import org.springframework.batch.item.ItemProcessor; | ||
import org.springframework.stereotype.Component; | ||
|
||
/** | ||
* <code>ContentsProcessedProcessor</code> performs any final work needed after a comic is | ||
* processed. | ||
* | ||
* @author Darryl L. Pierce | ||
*/ | ||
@Component | ||
@Log4j2 | ||
public class ContentsProcessedProcessor implements ItemProcessor<Comic, Comic> { | ||
@Override | ||
public Comic process(final Comic comic) { | ||
log.trace("Nothing to do"); | ||
return comic; | ||
} | ||
} |
45 changes: 45 additions & 0 deletions
45
...c/main/java/org/comixedproject/batch/comicbooks/processors/LoadFileContentsProcessor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
/* | ||
* ComiXed - A digital comic book library management application. | ||
* Copyright (C) 2021, The ComiXed Project | ||
* | ||
* This program is free software: you can redistribute it and/or modify | ||
* it under the terms of the GNU General Public License as published by | ||
* the Free Software Foundation, either version 3 of the License, or | ||
* (at your option) any later version. | ||
* | ||
* This program is distributed in the hope that it will be useful, | ||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
* GNU General Public License for more details. | ||
* | ||
* You should have received a copy of the GNU General Public License | ||
* along with this program. If not, see <http://www.gnu.org/licenses> | ||
*/ | ||
|
||
package org.comixedproject.batch.comicbooks.processors; | ||
|
||
import lombok.extern.log4j.Log4j2; | ||
import org.comixedproject.handlers.ComicFileHandler; | ||
import org.comixedproject.model.comic.Comic; | ||
import org.springframework.batch.item.ItemProcessor; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.stereotype.Component; | ||
|
||
/** | ||
* <code>LoadFileContentsProcessor</code> loads metadata for a comic. | ||
* | ||
* @author Darryl L. Pierce | ||
*/ | ||
@Component | ||
@Log4j2 | ||
public class LoadFileContentsProcessor implements ItemProcessor<Comic, Comic> { | ||
@Autowired private ComicFileHandler comicFileHandler; | ||
|
||
@Override | ||
public Comic process(final Comic comic) throws Exception { | ||
log.trace("Loading comic file contents"); | ||
this.comicFileHandler.loadComic(comic); | ||
log.trace("Returning updated comic"); | ||
return comic; | ||
} | ||
} |
49 changes: 49 additions & 0 deletions
49
...rc/main/java/org/comixedproject/batch/comicbooks/processors/LoadFileDetailsProcessor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
/* | ||
* ComiXed - A digital comic book library management application. | ||
* Copyright (C) 2021, The ComiXed Project | ||
* | ||
* This program is free software: you can redistribute it and/or modify | ||
* it under the terms of the GNU General Public License as published by | ||
* the Free Software Foundation, either version 3 of the License, or | ||
* (at your option) any later version. | ||
* | ||
* This program is distributed in the hope that it will be useful, | ||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
* GNU General Public License for more details. | ||
* | ||
* You should have received a copy of the GNU General Public License | ||
* along with this program. If not, see <http://www.gnu.org/licenses> | ||
*/ | ||
|
||
package org.comixedproject.batch.comicbooks.processors; | ||
|
||
import java.io.FileInputStream; | ||
import lombok.extern.log4j.Log4j2; | ||
import org.comixedproject.model.comic.Comic; | ||
import org.comixedproject.model.comic.ComicFileDetails; | ||
import org.comixedproject.utils.Utils; | ||
import org.springframework.batch.item.ItemProcessor; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.stereotype.Component; | ||
|
||
/** | ||
* <code>LoadFileDetailsProcessor</code> loads the file details for a comic. | ||
* | ||
* @author Darryl L. Pierce | ||
*/ | ||
@Component | ||
@Log4j2 | ||
public class LoadFileDetailsProcessor implements ItemProcessor<Comic, Comic> { | ||
@Autowired private Utils utils; | ||
|
||
@Override | ||
public Comic process(final Comic comic) throws Exception { | ||
log.trace("Creating file details container"); | ||
final ComicFileDetails fileDetails = new ComicFileDetails(comic); | ||
comic.setFileDetails(fileDetails); | ||
log.trace("Getting comic file hash"); | ||
fileDetails.setHash(this.utils.createHash(new FileInputStream(comic.getFilename()))); | ||
return comic; | ||
} | ||
} |
Oops, something went wrong.