Skip to content

Commit

Permalink
issue73 (#82)
Browse files Browse the repository at this point in the history 
* issue 73 -- refactor validation and initialization for emitters
  • Loading branch information
@tballison
tballison committed Nov 23, 2022
1 parent 3413405 commit 451b39a
Show file tree
Hide file tree
Showing 38 changed files with 1,659 additions and 1,011 deletions.
10 changes: 10 additions & 0 deletions NOTES.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Informal Notes

## H2
This is not a secure way of configuring H2, but for testing purposes, and IF you trust your ports, start a server:

```java -cp h2-2.1.214.jar org.h2.tools.Server -ifNotExists```

Then specify the url with any appended db, e.g. (on linux) ```jdbc:h2:tcp://192.168.200.4:9092/~/tika-test```

The *.mv.db file will be written in your home directory: `~/tika-test.mv.db`
1 change: 1 addition & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@
<excludes>
<exclude>CHANGES.txt</exclude>
<exclude>README.md</exclude>
<exclude>NOTES.md</exclude>
<exclude>.gitattributes</exclude>
</excludes>
</configuration>
Expand Down
106 changes: 53 additions & 53 deletions tika-gui-app/pom.xml
View file

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions tika-gui-app/src/main/java/org/tallison/tika/app/fx/AdvancedBatchController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,8 @@

public class AdvancedBatchController implements Initializable {

private static final Logger LOGGER = LogManager.getLogger(BatchInputController.class);
static AppContext APP_CONTEXT = AppContext.getInstance();
private static Logger LOGGER = LogManager.getLogger(BatchInputController.class);

@FXML
private ComboBox<String> digestOptions;

Expand Down
4 changes: 2 additions & 2 deletions tika-gui-app/src/main/java/org/tallison/tika/app/fx/BatchInputController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@

public class BatchInputController extends ControllerBase {

private static AppContext APP_CONTEXT = AppContext.getInstance();
private static Logger LOGGER = LogManager.getLogger(BatchInputController.class);
private static final AppContext APP_CONTEXT = AppContext.getInstance();
private static final Logger LOGGER = LogManager.getLogger(BatchInputController.class);

@FXML
private Button fsInputButton;
Expand Down
10 changes: 5 additions & 5 deletions tika-gui-app/src/main/java/org/tallison/tika/app/fx/BatchStatusController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@

public class BatchStatusController implements Initializable {

private static Map<String, PipesResult.STATUS> PIPES_STATUS_LOOKUP = new HashMap<>();
private static String UNPROCESSED_COLOR = "0066cc";
private static Map<PipesResult.STATUS, String> COLORS =
private static final Map<String, PipesResult.STATUS> PIPES_STATUS_LOOKUP = new HashMap<>();
private static final String UNPROCESSED_COLOR = "0066cc";
private static final Map<PipesResult.STATUS, String> COLORS =
Map.of(PipesResult.STATUS.PARSE_SUCCESS, "009900",
PipesResult.STATUS.PARSE_SUCCESS_WITH_EXCEPTION, "ffff00",
PipesResult.STATUS.EMIT_SUCCESS, "009900", PipesResult.STATUS.TIMEOUT, "ff9900",
Expand Down Expand Up @@ -216,7 +216,7 @@ private void addData(String name, double value) {
public void handle(MouseEvent e) {
pieSliceCaption.setTranslateX(e.getSceneX());
pieSliceCaption.setTranslateY(e.getSceneY());
pieSliceCaption.setText(String.valueOf(data.getPieValue()) + "%");
pieSliceCaption.setText(data.getPieValue() + "%");
}
});

Expand All @@ -234,7 +234,7 @@ private PipesResult.STATUS lookup(String name) {
}

private long countProcessed(AsyncStatus status) {
return status.getStatusCounts().values().stream().reduce(0l, Long::sum);
return status.getStatusCounts().values().stream().reduce(0L, Long::sum);
}
}

Expand Down
40 changes: 1 addition & 39 deletions tika-gui-app/src/main/java/org/tallison/tika/app/fx/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,51 +19,13 @@
import org.apache.tika.pipes.fetcher.fs.FileSystemFetcher;

public class Constants {
public static final String FS_EMITTER_CLASS =
"org.apache.tika.pipes.emitter.fs.FileSystemEmitter";
public static final String FS_FETCHER_CLASS = FileSystemFetcher.class.getName();

//an imaginary class
public static final String CSV_EMITTER_CLASS = "org.apache.tika.pipes.emitter.csv.CSVEmitter";

public static final String OPEN_SEARCH_EMITTER_CLASS =
"org.apache.tika.pipes.emitter.opensearch.OpenSearchEmitter";
public static final String JDBC_EMITTER_CLASS =
"org.apache.tika.pipes.emitter.jdbc.JDBCEmitter";
public static final String FS_FETCHER_CLASS = FileSystemFetcher.class.getName();

public static final String NO_DIGEST = "No Digest";

//JDBC emitter

// if the table doesn't exist, we create it via the dialog
// public static final String JDBC_TABLE_CREATED = "tableCreated";
public static final String JDBC_CONNECTION_STRING = "jdbcConnection";
public static final String JDBC_TABLE_NAME = "jdbcTableName";

public static final String JDBC_INSERT_SQL = "jdbcInsertString";
public static final String CSV_JDBC_INSERT_SQL = "csvInsertString";


//AbstractEmitterController
public static final String CSV_METADATA_PATH = "csvMetadataPath";

public static final String CSV_JDBC_CONNECTION_STRING = "csvSQLConnectionString";

public static final String CSV_DB_DIRECTORY = "csvDBDirectory";

//Used by filesystem emitter, fetcher and pipes iterator
//used as working directory for csv emitter
public static final String BASE_PATH = "basePath";


//opensearch
public static final String OPEN_SEARCH_URL = "openSearchUrl";
public static final String OPEN_SEARCH_USER = "userName";
public static final String OPEN_SEARCH_PW = "password";
public static final String OPEN_SEARCH_UPDATE_STRATEGY = "updateStrategy";

//csv emitter
public static final String CSV_DB_TABLE_NAME = "tika_table";
public static final String CSV_FILE_NAME = "csvFileName";

}
7 changes: 7 additions & 0 deletions tika-gui-app/src/main/java/org/tallison/tika/app/fx/ControllerBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,15 @@
import javafx.scene.layout.GridPane;
import javafx.scene.layout.Priority;
import javafx.scene.layout.Region;
import org.tallison.tika.app.fx.emitters.ValidationResult;

public class ControllerBase {

public static void alert(ValidationResult validationResult) {
alert(validationResult.getTitle().get(), validationResult.getHeader().get(),
validationResult.getMsg().get());
}

public static void alert(String title, String header, String content) {
Alert alert = new Alert(Alert.AlertType.WARNING);
alert.setTitle(title);
Expand Down
2 changes: 1 addition & 1 deletion tika-gui-app/src/main/java/org/tallison/tika/app/fx/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

public class Main {

private static Logger LOGGER = LogManager.getLogger(Main.class);
private static final Logger LOGGER = LogManager.getLogger(Main.class);

public static void main(String[] args) throws Exception {
String javaHome = System.getProperty("TIKA_GUI_JAVA_HOME");
Expand Down
92 changes: 79 additions & 13 deletions tika-gui-app/src/main/java/org/tallison/tika/app/fx/TikaController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;

import javafx.event.ActionEvent;
import javafx.event.EventHandler;
Expand All @@ -26,6 +27,8 @@
import javafx.scene.Scene;
import javafx.scene.control.Alert;
import javafx.scene.control.Button;
import javafx.scene.control.ButtonBar;
import javafx.scene.control.ButtonType;
import javafx.scene.control.Label;
import javafx.scene.control.ProgressIndicator;
import javafx.scene.control.TabPane;
Expand All @@ -42,12 +45,15 @@
import org.tallison.tika.app.fx.batch.BatchProcessConfig;
import org.tallison.tika.app.fx.config.ConfigItem;
import org.tallison.tika.app.fx.ctx.AppContext;
import org.tallison.tika.app.fx.emitters.EmitterSpec;
import org.tallison.tika.app.fx.emitters.ValidationResult;
import org.tallison.tika.app.fx.status.StatusUpdater;
import org.tallison.tika.app.fx.utils.OptionalUtil;

public class TikaController extends ControllerBase {

private static final Logger LOGGER = LogManager.getLogger(TikaController.class);
static AppContext APP_CONTEXT = AppContext.getInstance();
private static Logger LOGGER = LogManager.getLogger(TikaController.class);
@FXML
private Label welcomeText;

Expand Down Expand Up @@ -185,8 +191,58 @@ public void configureOutput(ActionEvent actionEvent) throws IOException {
final Stage stage = new Stage();
stage.setTitle("Select Output");
stage.setScene(scene);
stage.setOnCloseRequest(new EventHandler<WindowEvent>() {
public void handle(WindowEvent we) {
onEmitterClose(we, stage);
}
});

stage.showAndWait();
updateButtons(BatchProcess.STATUS.READY);

}

private void onEmitterClose(WindowEvent windowEvent, Stage stage) {

if (APP_CONTEXT.getBatchProcessConfig().isEmpty()) {
windowEvent.consume();
return;
}
if (APP_CONTEXT.getBatchProcessConfig().get().getEmitter().isEmpty()) {
windowEvent.consume();
return;
}
EmitterSpec emitterSpec = APP_CONTEXT.getBatchProcessConfig().get().getEmitter().get();
if (emitterSpec.isValid()) {
windowEvent.consume();
updateButtons(BatchProcess.STATUS.READY);
return;
}
AtomicBoolean close = new AtomicBoolean(true);
Alert alert = new Alert(Alert.AlertType.CONFIRMATION);
alert.setTitle("Output Not Fully Configured");
if (!OptionalUtil.isEmpty(emitterSpec.getNotValidMessage())) {
alert.setContentText(emitterSpec.getNotValidMessage().get());
} else {
alert.setContentText("");
}
ButtonType goBack = new ButtonType("Go Back to Configuration", ButtonBar.ButtonData.YES);
ButtonType ignore = new ButtonType("Ignore", ButtonBar.ButtonData.NO);
//ButtonType cancelButton = new ButtonType("Cancel", ButtonBar.ButtonData.CANCEL_CLOSE);
alert.getButtonTypes().setAll(ignore, goBack);
alert.showAndWait().ifPresent(type -> {
if (type.getText().startsWith("Go Back")) {
close.set(false);
} else if (type.getText().startsWith("Ignore")) {
close.set(true);
}
});
//go back to where we were
windowEvent.consume();
if (close.get()) {
stage.close();
}

}

public void updateButtons(BatchProcess.STATUS status) {
Expand All @@ -208,17 +264,15 @@ public void updateButtons(BatchProcess.STATUS status) {
configureAdvanced.setDisable(false);
cancelButton.setDisable(true);

if (fullyConfigured()) {
runButton.setDisable(false);
} else {
runButton.setDisable(true);
}
runButton.setDisable(!fullyConfigured());
}

private boolean fullyConfigured() {
BatchProcessConfig config = APP_CONTEXT.getBatchProcessConfig().get();
return config.getEmitter().isPresent() && config.getPipesIterator().isPresent() &&
config.getEmitter().isPresent();
return config.getFetcher().isPresent() && config.getFetcher().get().isValid() &&
config.getPipesIterator().isPresent() &&
config.getPipesIterator().get().isValid() && config.getEmitter().isPresent() &&
config.getEmitter().get().isValid();
}

@FXML
Expand All @@ -233,6 +287,19 @@ public void runTika(ActionEvent actionEvent) throws Exception {
}
if (APP_CONTEXT.getBatchProcessConfig().isEmpty()) {
LOGGER.warn("batch processConfig must not be empty!");
return;
}

BatchProcessConfig bpc = APP_CONTEXT.getBatchProcessConfig().get();
//TODO -- validate as much as possible
EmitterSpec emitterSpec = bpc.getEmitter().get();
ValidationResult result = emitterSpec.initialize();
//invalidate emitter so that run button is turned off --
//this forces the user to have to reconfigure the emitters
emitterSpec.setValid(false);
if (result != ValidationResult.OK) {
alert(result.getTitle().get(), result.getHeader().get(), result.getMsg().get());
return;
}
//TODO -- all sorts of checks
//Is there already a batch process.
Expand Down Expand Up @@ -282,14 +349,13 @@ public void showFetcher(MouseEvent mouseEvent) {
}

public void showEmitter(MouseEvent mouseEvent) {
Optional<ConfigItem> configItem = APP_CONTEXT.getBatchProcessConfig().get().getEmitter();
if (configItem.isPresent()) {
String fullLabel = configItem.get().getAttributes().get("basePath");
Optional<EmitterSpec> emitterSpec = APP_CONTEXT.getBatchProcessConfig().get().getEmitter();
if (emitterSpec.isPresent()) {
Alert alert = new Alert(Alert.AlertType.INFORMATION);
alert.setTitle("Emitter");
alert.setHeaderText(configItem.get().getShortLabel());
alert.setHeaderText(emitterSpec.get().getShortLabel().get());
alert.setResizable(true);
alert.setContentText(configItem.get().getFullLabel());
alert.setContentText(emitterSpec.get().getFullLabel().get());
alert.getDialogPane().setMinWidth(500);
alert.getDialogPane().setMinHeight(Region.USE_PREF_SIZE);
alert.showAndWait();
Expand Down
37 changes: 20 additions & 17 deletions tika-gui-app/src/main/java/org/tallison/tika/app/fx/batch/BatchProcess.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.tallison.tika.app.fx.config.TikaConfigWriter;
import org.tallison.tika.app.fx.csv.CSVEmitterHelper;
import org.tallison.tika.app.fx.ctx.AppContext;
import org.tallison.tika.app.fx.status.MutableStatus;
import org.tallison.tika.app.fx.status.StatusUpdater;
Expand All @@ -50,23 +49,23 @@

public class BatchProcess {

private static Logger LOGGER = LogManager.getLogger(BatchProcess.class);
private static final Logger LOGGER = LogManager.getLogger(BatchProcess.class);
private final MutableStatus mutableStatus = new MutableStatus(STATUS.READY);
private long runningProcessId = -1;
private Path configFile;
private BatchRunner batchRunner = null;
private BatchProcessConfig batchProcessConfig = null;
private Optional<Exception> jvmException = Optional.empty();
private Optional<String> jvmErrorMsg = Optional.empty();
private ObjectMapper objectMapper =
private final ObjectMapper objectMapper =
JsonMapper.builder().addModule(new JavaTimeModule()).build();
private ExecutorService daemonExecutorService = Executors.newFixedThreadPool(2, r -> {
private final ExecutorService daemonExecutorService = Executors.newFixedThreadPool(2, r -> {
Thread t = Executors.defaultThreadFactory().newThread(r);
t.setDaemon(true);
return t;
});
private ExecutorCompletionService<Integer> executorCompletionService =
private final ExecutorCompletionService<Integer> executorCompletionService =
new ExecutorCompletionService<>(daemonExecutorService);
private long runningProcessId = -1;
private Path configFile;
private BatchRunner batchRunner = null;
private BatchProcessConfig batchProcessConfig = null;
private Optional<Exception> jvmException = Optional.empty();
private Optional<String> jvmErrorMsg = Optional.empty();

public synchronized void start(BatchProcessConfig batchProcessConfig,
StatusUpdater statusUpdater) throws TikaException, IOException {
Expand Down Expand Up @@ -130,7 +129,9 @@ public synchronized void cancel() {
}
daemonExecutorService.shutdownNow();
try {
CSVEmitterHelper.cleanCSVTempResources(batchProcessConfig.getEmitter().get());
if (batchProcessConfig.getEmitter().isPresent()) {
batchProcessConfig.getEmitter().get().close();
}
} catch (IOException e) {
LOGGER.warn("Failed to delete csv tmp");
}
Expand Down Expand Up @@ -199,7 +200,7 @@ public Optional<String> getJvmErrorMsg() {
}

public enum STATUS {
READY, ERROR, RUNNING, COMPLETE, CANCELED;
READY, ERROR, RUNNING, COMPLETE, CANCELED
}

private enum PROCESS_ID {
Expand All @@ -219,6 +220,8 @@ public BatchRunner(Path tikaConfig, BatchProcessConfig batchProcessConfig) {
@Override
public Integer call() throws Exception {
List<String> commandLine = buildCommandLine();


process = new ProcessBuilder(commandLine).inheritIO() //TODO -- for dev purposes only
.start();
mutableStatus.set(STATUS.RUNNING);
Expand Down Expand Up @@ -261,7 +264,9 @@ public Integer call() throws Exception {
jvmErrorMsg = Optional.of(msg);
mutableStatus.set(STATUS.ERROR);
} else {
CSVEmitterHelper.writeCSV(AppContext.getInstance());
if (batchProcessConfig.getEmitter().isPresent()) {
batchProcessConfig.getEmitter().get().close();
}
mutableStatus.set(STATUS.COMPLETE);
}
return PROCESS_ID.BATCH_PROCESS.ordinal();
Expand Down Expand Up @@ -295,9 +300,7 @@ private String buildClassPath() {
sb.append(File.pathSeparator);
sb.append(ProcessUtils.escapeCommandLine(
AppContext.TIKA_EXTRAS_BIN_PATH.toAbsolutePath() + "/*"));
sb.append(File.pathSeparator);
//TODO refactor batch process config to generate class path
//for fetchers/emitters

batchProcessConfig.appendPipesClasspath(sb);
return sb.toString();
}
Expand Down
Loading

0 comments on commit 451b39a

Please sign in to comment.