diff --git a/.idea/misc.xml b/.idea/misc.xml index e9aadba..a9a9dd0 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -5,6 +5,7 @@ diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 94a25f7..35eb1dd 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/README.md b/README.md index afd85ec..419a1c5 100644 --- a/README.md +++ b/README.md @@ -4,23 +4,23 @@ ## Introduction -**OpenLabeler** is an open source application for annotating objects. It can generate the PASCAL VOC format XML annotation file for artificial intelligence and deep learning training. A unique aspect of this application is its ability to use inference (with [TensorFlow](https://www.tensorflow.org)) to help improve accuracy and speed up the annotation process. +**OpenLabeler** is an open-source application for annotating objects. It can generate the PASCAL VOC format XML annotation file for artificial intelligence and deep learning training. This application's unique aspect is its ability to use inference (with [TensorFlow](https://www.tensorflow.org)) to improve accuracy and speed up the annotation process. -OpenLabeler is written in [OpenJDK](https://openjdk.java.net)/[OpenJFX](https://openjfx.io) (version 18.x). +OpenLabeler is written in [OpenJDK](https://openjdk.java.net)/[OpenJFX](https://openjfx.io) (version 21.x). ![Application](assets/app.png) A few highlights: -* Fast Labeling (no need for Open/Save File actions) +* Fast labeling (no need for Open/Save File actions) ![General Preferences](assets/pref-general.png) * Multi-level undo/redo * Annotation "hints" (using TensorFlow inference) -* Pre-built installation packages for macOS (tested on macOS Mojave), Linux (tested on Ubuntu 18.04 LTS), and Windows (tested on Windows 10 Pro) +* Pre-built installation packages for macOS (tested on macOS Sonoma), Linux (tested on Ubuntu 20.04 LTS), and Windows (tested on Windows 10 Pro) ## Inference -OpenLabeler can help improve the speed and accuracy of annotation by providing labeling "hints" from a saved model using TensorFlow. +OpenLabeler can help improve the speed and accuracy of annotation by offering labeling "hints" from a saved model using TensorFlow (currently, only x86/x86_64 machines are supported). For example, you have thousands of images to annotate. After labeling the first 300 or so images, you could train a model using these 300 samples, then configure OpenLabeler to use this intermediary model to give you labeling suggestions for the remaining images, thereby speeding up the annotation task. @@ -32,6 +32,8 @@ The **Saved Model Location** is the *folder* where the `.pb` file is located. If OpenLabeler supports graphs with the `image_tensor` and `encoded_image_string_tensor` operations/input types. +The protobuf sources is located in https://github.com/tensorflow/models/tree/master/research/object_detection/protos + ## Training Support *Note: This is currently an experimental feature.* @@ -39,13 +41,11 @@ OpenLabeler supports graphs with the `image_tensor` and `encoded_image_string_te OpenLabeler can be used to start/stop a training process in TensorFlow running inside a [Docker](https://www.docker.com) container. Containers with [TensorFlow 2](https://www.tensorflow.org/install/docker) and [Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) dependencies have been pre-built for your convenience. To use this feature: 1. [Install Docker](https://docs.docker.com/install) on your host machine -2. *Windows only* Open Docker > Settings > General and check "Expose daemon on tcp://localhost:2375 without TLS" -3. *Windows only* Create a `\.docker-java.properties` file. Add a `DOCKER_HOST=tcp://localhost:2375 entry`. See [reference](https://github.com/docker-java/docker-java) -4. Choose a pre-built, `kinhong/openlabeler:tf-2.3.1` or `kinhong/openlabeler:tf-2.3.1-gpu`, [docker image](https://cloud.docker.com/repository/docker/kinhong/openlabeler/tags) from [Docker Hub](https://hub.docker.com/) and pull it to your docker host -5. Download a base model from the [TensorFlow 2 Detection Model Zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) for transfer learning -6. Configure the Training Preference settings (and add the label map entries) +2. Choose a pre-built, `kinhong/openlabeler:tf-2.3.1` or `kinhong/openlabeler:tf-2.3.1-gpu`, [docker image](https://cloud.docker.com/repository/docker/kinhong/openlabeler/tags) from [Docker Hub](https://hub.docker.com/) and pull it to your docker host +3. Download a base model from the [TensorFlow 2 Detection Model Zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) for transfer learning +4. Configure the Training Preference settings (and add the label map entries) ![Train Preferences](assets/pref-train.png) -7. You can then start, stop, continue, restart training, or export the inference graph +5. You can then start, stop, continue, restart training, or export the inference graph ## Shortcut Keys @@ -105,7 +105,7 @@ First make sure the environment variable JAVA_HOME has been set accordingly ### macOS -1. Download and install [OpenJDK 18](http://jdk.java.net/18) +1. Download and install [OpenJDK 21](http://jdk.java.net/21) 2. Download and install [Maven](https://maven.apache.org/install.html) ``` cd @@ -116,12 +116,12 @@ The macOS .pkg installer can be found under the app/target/package directory. ### Linux ``` sudo add-apt-repository ppa:openjdk-r/ppa \ -sudo apt-get update -q \ -sudo apt install -y openjdk-18-jdk - -sudo apt-get install maven +sudo apt update -q \ +sudo apt install -y openjdk-21-jdk -sudo apt-get install fakeroot +sudo apt install maven +sudo apt install binutils +sudo apt install fakeroot cd mvn clean package -Drevision=x.y.z @@ -130,8 +130,10 @@ The Linux .deb bundle can be found under the app/target/package directory. ### Windows -1. Download [OpenJDK 18](http://jdk.java.net/18/) for Windows and unzip to a directory with no spaces (e.g., `C:\java\jdk-18`) +1. Download [OpenJDK 21](https://learn.microsoft.com/en-us/java/openjdk/download#openjdk-21) for Windows and unzip to a directory with no spaces (e.g., `C:\java\jdk-21`) 2. Download [Maven](https://maven.apache.org/download.cgi) and unzip to a directory with no spaces (e.g., `C:\java\apache-maven`) +3. Download [Wix Toolset](https://github.com/wixtoolset/wix3) and unzip to a directory with no spaces (e.g., `c:\wix`) +3. Make sure java, mvn and wix executables are in your Windows PATH (e.g., `set PATH=%PATH%;C:\java\jdk-21\bin;C:\java\apache-maven\bin;c:\wix`) ```DOS .bat cd diff --git a/app/pom.xml b/app/pom.xml index 496d60e..95a2b02 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -56,19 +56,13 @@ org.tensorflow - tensorflow-core-platform - 0.4.1 + tensorflow-core-api + 0.5.0 org.tensorflow proto 1.15.0 - - - protobuf-java - com.google.protobuf - - com.google.protobuf @@ -107,7 +101,7 @@ com.github.docker-java - docker-java + docker-java-core 3.3.6 @@ -198,6 +192,7 @@ maven-resources-plugin 3.3.1 + UTF-8 png icns @@ -206,6 +201,7 @@ + org.apache.maven.plugins maven-dependency-plugin 3.6.1 @@ -218,19 +214,6 @@ ${project.build.directory}/libs runtime - org.openjfx - - - - copy-modules - prepare-package - - copy-dependencies - - - ${project.build.directory}/mods - runtime - org.openjfx @@ -238,9 +221,6 @@ - - java.base,java.logging,java.desktop,java.prefs,java.xml,javafx.base,javafx.graphics,javafx.controls,javafx.fxml,javafx.web,javafx.media,javafx.swing - OpenLabeler @@ -251,6 +231,14 @@ mac + + + org.tensorflow + tensorflow-core-api + 0.5.0 + macosx-x86_64 + + @@ -304,8 +292,6 @@ --module-path ${java.home}/jmods:${project.build.directory}/mods - --add-modules - ${project.build.add-modules} @@ -325,6 +311,14 @@ Linux + + + org.tensorflow + tensorflow-core-api + 0.5.0 + linux-x86_64 + + @@ -376,8 +370,6 @@ --module-path ${java.home}/jmods:${project.build.directory}/mods - --add-modules - ${project.build.add-modules} @@ -396,6 +388,14 @@ windows + + + org.tensorflow + tensorflow-core-api + 0.5.0 + windows-x86_64 + + @@ -448,8 +448,6 @@ --module-path ${java.home}/jmods;${project.build.directory}/mods - --add-modules - ${project.build.add-modules} diff --git a/app/src/main/java/com/easymobo/openlabeler/tensorflow/ObjectDetector.java b/app/src/main/java/com/easymobo/openlabeler/tensorflow/ObjectDetector.java index f63fb89..c742278 100644 --- a/app/src/main/java/com/easymobo/openlabeler/tensorflow/ObjectDetector.java +++ b/app/src/main/java/com/easymobo/openlabeler/tensorflow/ObjectDetector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022. Kin-Hong Wong. All Rights Reserved. + * Copyright (c) 2024. Kin-Hong Wong. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,15 +26,16 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.ArrayUtils; +import org.tensorflow.Result; import org.tensorflow.SavedModelBundle; import org.tensorflow.Tensor; import org.tensorflow.TensorFlow; -import org.tensorflow.framework.DataType; -import org.tensorflow.framework.MetaGraphDef; -import org.tensorflow.framework.SignatureDef; -import org.tensorflow.framework.TensorInfo; import org.tensorflow.ndarray.Shape; import org.tensorflow.ndarray.buffer.DataBuffers; +import org.tensorflow.proto.framework.DataType; +import org.tensorflow.proto.framework.MetaGraphDef; +import org.tensorflow.proto.framework.SignatureDef; +import org.tensorflow.proto.framework.TensorInfo; import org.tensorflow.types.TFloat32; import org.tensorflow.types.TString; import org.tensorflow.types.TUint8; @@ -143,7 +144,7 @@ else if (!savedModelFile.exists() && path == null) { LOG.info(savedModelFile.toString() + " does not exist"); } } - catch (Exception ex) { + catch (Throwable ex) { LOG.log(Level.SEVERE, "Unable to update " + path, ex); } return null; @@ -156,7 +157,7 @@ public List detect(File imageFile) throws IOException { return Collections.emptyList(); } List hints = new ArrayList(); - List outputs; + Result result; Tensor input = null; String operation = ""; BufferedImage img = ImageIO.read(imageFile); @@ -178,7 +179,7 @@ else if (sig.containsInputs("input_tensor")) { operation = tensorInfo.getName(); input = tensorInfo.getDtype() == DataType.DT_STRING ? makeImageStringTensor(imageFile) : makeImageTensor(img); } - outputs = model.session() + result = model.session() .runner() .feed(operation, input) .fetch(sig.getOutputsOrThrow("detection_scores").getName()) @@ -191,9 +192,9 @@ else if (sig.containsInputs("input_tensor")) { input.close(); } } - try (TFloat32 scoresT = (TFloat32)outputs.get(0); - TFloat32 classesT = (TFloat32)outputs.get(1); - TFloat32 boxesT = (TFloat32)outputs.get(2)) { + try (TFloat32 scoresT = (TFloat32)result.get(0); + TFloat32 classesT = (TFloat32)result.get(1); + TFloat32 boxesT = (TFloat32)result.get(2)) { // All these tensors have: // - 1 as the first dimension // - maxObjects as the second dimension diff --git a/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFRecordCreator.java b/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFRecordCreator.java index 9456697..6adad1a 100644 --- a/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFRecordCreator.java +++ b/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFRecordCreator.java @@ -24,8 +24,8 @@ import jakarta.xml.bind.Unmarshaller; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.IOUtils; -import org.tensorflow.example.*; import org.tensorflow.hadoop.util.TFRecordWriter; +import org.tensorflow.proto.example.*; import java.io.DataOutputStream; import java.io.File; diff --git a/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFTrainer.java b/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFTrainer.java index d2d20c3..c5faa0d 100644 --- a/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFTrainer.java +++ b/app/src/main/java/com/easymobo/openlabeler/tensorflow/TFTrainer.java @@ -23,8 +23,8 @@ import com.github.dockerjava.api.command.CreateContainerResponse; import com.github.dockerjava.api.model.*; import com.github.dockerjava.core.DefaultDockerClientConfig; -import com.github.dockerjava.core.DockerClientBuilder; import com.github.dockerjava.core.DockerClientConfig; +import com.github.dockerjava.core.DockerClientImpl; import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; import com.github.dockerjava.transport.DockerHttpClient; import com.google.protobuf.TextFormat; @@ -43,12 +43,10 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.ResourceBundle; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -62,18 +60,16 @@ public class TFTrainer implements AutoCloseable private static final Logger LOG = Logger.getLogger(MethodHandles.lookup().lookupClass().getCanonicalName()); private static final String EXPORTER = "OpenLabeler-Exporter"; - private ResourceBundle bundle = ResourceBundle.getBundle("bundle"); - // Monitors TF training status private WatchService watcher; private WatchKey tfTrainDirWatchKey; // Docker - private static DockerClientConfig dockerClientConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); - private static DockerHttpClient dockerHttpClient = new ApacheDockerHttpClient.Builder().dockerHost(dockerClientConfig.getDockerHost()).build(); - private static DockerClient dockerClient = DockerClientBuilder.getInstance().withDockerHttpClient(dockerHttpClient).build(); + private static final DockerClientConfig dockerClientConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); + private static final DockerHttpClient dockerHttpClient = new ApacheDockerHttpClient.Builder().dockerHost(dockerClientConfig.getDockerHost()).build(); + private static final DockerClient dockerClient = DockerClientImpl.getInstance(dockerClientConfig, dockerHttpClient); - private SimpleIntegerProperty checkpointProperty = new SimpleIntegerProperty(-1); + private final SimpleIntegerProperty checkpointProperty = new SimpleIntegerProperty(-1); public IntegerProperty checkpointProperty() { return checkpointProperty; } @@ -87,14 +83,12 @@ public void init() { } watch(getModelDirPath(Settings.getTFDataDir())); - Settings.tfDataDirProperty.addListener((observable, oldValue, newValue) -> { - watch(Paths.get(newValue)); - }); + Settings.tfDataDirProperty.addListener((observable, oldValue, newValue) -> watch(Paths.get(newValue))); } @Override public void close() { - Optional.ofNullable(tfTrainDirWatchKey).ifPresent(key -> key.cancel()); + Optional.ofNullable(tfTrainDirWatchKey).ifPresent(WatchKey::cancel); IOUtils.closeQuietly(watcher); } @@ -174,11 +168,11 @@ public static List getLabelMapItems(String dataDir) { public static List getLabelMapItems(Path path) { try { if (path.toFile().exists()) { - String text = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + String text = Files.readString(path); StringIntLabelMap.Builder builder = StringIntLabelMap.newBuilder(); TextFormat.merge(text, builder); StringIntLabelMap proto = builder.build(); - return proto.getItemList().stream().map(i -> new LabelMapItem(i)).collect(Collectors.toList()); + return proto.getItemList().stream().map(LabelMapItem::new).collect(Collectors.toList()); } } catch (Exception ex) { @@ -211,7 +205,9 @@ public static void createTrainData(List items) { try { FileUtils.deleteDirectory(getDataPath().toFile()); String dataDir = Settings.getTFDataDir(); - getDataPath().toFile().mkdirs(); + if (!getDataPath().toFile().exists() && !getDataPath().toFile().mkdirs()) { + throw new Exception("Unable to create " + dataDir); + } saveLabelMap(items, dataDir); LOG.info("Creating training data in " + dataDir + "..."); TFRecordCreator recordCreator = new TFRecordCreator(Paths.get(Settings.getTFImageDir()), Paths.get(Settings.getTFAnnotationDir()), getDataPath()); @@ -261,22 +257,23 @@ public static void exportGraph(int checkpoint) { } public static void saveLabelMap(List items, String dataDir) throws IOException { - if (items.size() <= 0) { + if (items.isEmpty()) { LOG.severe("No label map items"); } StringIntLabelMapOuterClass.StringIntLabelMap.Builder builder = StringIntLabelMapOuterClass.StringIntLabelMap.newBuilder(); - items.forEach(item -> { - builder.addItem(StringIntLabelMapItem.newBuilder().setId(item.getId()).setName(item.getName()).setDisplayName(item.getDisplayName())); - }); + items.forEach(item -> builder.addItem(StringIntLabelMapItem.newBuilder().setId(item.getId()).setName(item.getName()).setDisplayName(item.getDisplayName()))); String labelMap = TextFormat.printToString(builder.build()); if (StringUtils.isNotEmpty(labelMap)) { - getLabelMapPath(dataDir).toFile().getParentFile().mkdirs(); + File parentFile = getLabelMapPath(dataDir).toFile().getParentFile(); + if (!parentFile.exists() && !parentFile.mkdirs()) { + throw new IOException("Unable to create " + parentFile.getAbsolutePath()); + } Files.write(getLabelMapPath(dataDir), labelMap.getBytes()); LOG.info("Created " + getLabelMapPath(dataDir)); } } - public static int getTrainCkpt(String baseModelDir) { + public static int getTrainCkpt(String baseModelDir) throws NullPointerException { File trainDir = getModelDirPath(baseModelDir).toFile(); if (!trainDir.exists()) { return -1; @@ -287,7 +284,7 @@ public static int getTrainCkpt(String baseModelDir) { for (File file : trainDir.listFiles()) { Matcher matcher = pattern.matcher(file.getName()); if (matcher.matches()) { - ckpt = Math.max(ckpt, Integer.valueOf(matcher.group(1))); + ckpt = Math.max(ckpt, Integer.parseInt(matcher.group(1))); } } return ckpt; @@ -403,7 +400,7 @@ public void save() throws IOException { private Pipeline.TrainEvalPipelineConfig parse(Path path) { try { - String text = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + String text = Files.readString(path); Pipeline.TrainEvalPipelineConfig.Builder builder = Pipeline.TrainEvalPipelineConfig.newBuilder(); TextFormat.Parser parser = TextFormat.Parser.newBuilder().build(); @@ -441,7 +438,7 @@ public int getNumTrainSteps() { // Update pipeline config with paths expected in docker container private Pipeline.TrainEvalPipelineConfig update(Pipeline.TrainEvalPipelineConfig config) { if (config == null) { - return config; + return null; } // train_config.fine_tune_checkpoint Pipeline.TrainEvalPipelineConfig.Builder builder = config.toBuilder(); diff --git a/app/src/main/java/module-info.java b/app/src/main/java/module-info.java new file mode 100644 index 0000000..43077d6 --- /dev/null +++ b/app/src/main/java/module-info.java @@ -0,0 +1,32 @@ +module com.easymobo.openlabeler.app { + requires javafx.controls; + requires javafx.fxml; + requires javafx.media; + requires javafx.web; + requires javafx.swing; + requires org.apache.commons.io; + requires java.logging; + requires jakarta.xml.bind; + requires org.apache.commons.lang3; + requires org.kordamp.ikonli.javafx; + requires org.kordamp.ikonli.materialdesign; + requires org.apache.commons.collections4; + requires org.apache.commons.codec; + requires easybind; + requires opencv; + requires com.fasterxml.jackson.databind; + requires java.prefs; + requires reactfx; + requires org.fxmisc.undo; + requires proto; + requires com.github.dockerjava.core; + requires com.github.dockerjava.api; + requires com.github.dockerjava.transport; + requires com.github.dockerjava.transport.httpclient5; + requires org.tensorflow.core.api; + requires org.tensorflow.ndarray; + requires com.google.protobuf; + requires org.tensorflow; + + exports com.easymobo.openlabeler; +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 713c054..3e98210 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ maven-compiler-plugin 3.12.1 - 21 + 18 diff --git a/preloader/pom.xml b/preloader/pom.xml index c63cea5..c1c934e 100644 --- a/preloader/pom.xml +++ b/preloader/pom.xml @@ -47,7 +47,7 @@ maven-compiler-plugin 3.12.1 - 21 + 18