external plugins

README.md

@@ -1,5 +1,5 @@
 # Frankenstein VR
-Video Stream Analysis and Manipulation Framework for Java, where custom filters can be simply added into the processing pipeline.
+Video Stream Analysis and Manipulation Framework for Java and C++, where custom filters can be simply added into the processing pipeline.
 
 The Tool supports 
 * [OpenCV](http://www.opencv.org/releases.html) - computer vision and machine learning
@@ -9,31 +9,23 @@ The Tool supports
 
 <img src="doc/pipeline.png" width="100%">
 
-The pipeline allows input as video file, camera, network stream or pictures stored as left/right 3D slides (e.g. from nikon camera).
-
-Frankenstein VR can e.g. be used to convert anaglyph or side-by-side 3D videos for VR display,
- and the output video can be perfectly viewed in 3D with VR Hardware and a video players like [LittlStar](http://littlstar.info). 
-VR videos appear like displayed on a virtual 160-inch curved 3D display in front of you.
-
-3D is optional. Hence, Frankenstein VR can be used solely for classic video processing. It focuses on filters, that are not part of common tools.
-
-
 ## Screenshots
 <img src="doc/config.png" width="45%"> <img src="doc/processing.png" width="45%" />
 
 
 ## Samples
 I have uploaded some samples to vimeo: <a href="https://vimeo.com/user68089135"><img src="doc/vimeo.png"/></a>
 
-## Basic Features
-Frankenstein VR is an experimental video converter with some video filters/features:
+## Features and video filters in the main pipeline
 - Virtual Reality side-by-side converter (projection, padding, shrinking)
 - Anaglyph (e.g. red/blue) to grayscale side-by-side converter
 - 3D Slideshow (SBS Video from 3D image pairs; see [samples](doc/slides) )
 - Over/Under to Left/Right (side-by-side) converter
 - Left/Right side swapper
 - Test Image (good for calibrating configurations on a VR display)
-Some filters can be chained together.
+
+### 3D / VR Features
+The pipeline allows input as video file, camera, network stream or pictures stored as left/right 3D slides (e.g. from nikon camera). Frames can be converted tp anaglyph or side-by-side 3D videos for VR display, and the output video can be perfectly viewed in 3D with VR Hardware and a video players like [LittlStar](http://littlstar.info). VR videos appear in this viewer like displayed on a virtual 160-inch curved 3D display in front of you.
 
 ## Status
 
@@ -70,8 +62,8 @@ FFMPEG build contains H264 encoder based on the OpenH264 library, that should be
   - e.g.: [Pre-Release 0.1](https://github.com/olir/Frankenstein/releases/download/0.1/launch.jnlp)
 3. Accept warnings and execute.
 
-##For developers##
-When you work on custom filters, you can concentrate on manipulating images with the OpenCV library.  For more details read   [SegmentFilters](https://github.com/olir/Frankenstein/blob/master/doc/SegmentFilters.md).
+## For developers ##
+When you work on custom filters, you can concentrate on manipulating images with the OpenCV or JogAmp libraries.  For more details read   [SegmentFilters](https://github.com/olir/Frankenstein/blob/master/doc/SegmentFilters.md).
 
 
 

app/src/main/java/de/screenflow/frankenstein/Main.java

@@ -17,6 +17,8 @@ public static void main(String[] args) {
 
 		if (args.length==0 || c.isVisual()) {
 
+			de.screenflow.frankenstein.fxml.FxMain.fxmain(c);
+			/*
 			Class<?> fxMain;
 			try {
 				fxMain = Class.forName("de.screenflow.frankenstein.fxml.FxMain");
@@ -28,7 +30,7 @@ public static void main(String[] args) {
 					| InvocationTargetException | NoSuchMethodException | SecurityException e) {
 				e.printStackTrace();
 			}
-
+			 */
 		}
 	}
 }

app/src/main/java/de/screenflow/frankenstein/fxml/FxMain.java

@@ -15,12 +15,17 @@
  */
 package de.screenflow.frankenstein.fxml;
 
+import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.PropertyResourceBundle;
 import java.util.ResourceBundle;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.net.URLClassLoader;
 
 import de.screenflow.frankenstein.Configuration;
@@ -29,6 +34,7 @@
 import de.screenflow.frankenstein.vf.segment.BWFilter;
 import de.screenflow.frankenstein.vf.segment.GLExampleFilter;
 import de.screenflow.frankenstein.vf.segment.NativeExampleFilter;
+import de.screenflow.frankenstein.vf.segment.NativeSegmentFilter;
 import de.screenflow.frankenstein.vf.segment.StereoDistanceFilter;
 import de.screenflow.frankenstein.vf.segment.VideoEqualizerFilter;
 import javafx.application.Application;
@@ -187,18 +193,64 @@ public List<SegmentVideoFilter> getLocalFilters() {
 
 	public void createSegmentFilters() {
 		segmentFilters = new ArrayList<SegmentVideoFilter>();
-		segmentFilters.add(new BWFilter());
-		segmentFilters.add(new StereoDistanceFilter());
-		segmentFilters.add(new SampleFilter());
-		segmentFilters.add(new GLExampleFilter());
-
+
 		try {
-			segmentFilters.add(new NativeExampleFilter()); // try to load from plugin jar
-			segmentFilters.add(new VideoEqualizerFilter()); // try to load from plugin jar
+			segmentFilters.add(new BWFilter());
+			segmentFilters.add(new StereoDistanceFilter());
+			segmentFilters.add(new SampleFilter());
+			segmentFilters.add(new GLExampleFilter());
+		} catch (Throwable t) {
+			t.printStackTrace();
 		}
-		catch(Throwable t) {
+
+		// Filters with native proxy in jar
+		try {
+			segmentFilters.add(new NativeExampleFilter()); // try to load from
+															// plugin jar
+			segmentFilters.add(new VideoEqualizerFilter()); // try to load from
+															// plugin jar
+		} catch (Throwable t) {
 			t.printStackTrace();
 		}
+
+		// Filters completly in jar
+		try {
+			segmentFilters.add(loadExternalFilterInstance("de.screenflow.frankenstein.vf.external.ExternalSampleFilter"));
+		} catch (Throwable t) {
+			t.printStackTrace();
+		}
+
+	}
+
+	private SegmentVideoFilter loadExternalFilterInstance(String filterClassName) {
+		try {
+			// use dynamic loading and reflection when loading jni proxy class
+			// from jar, so app do not depend on it.
+			URLClassLoader childLoader = getLoader();
+			Class filterClass = Class.forName(filterClassName, true, childLoader);
+			SegmentVideoFilter filter = (SegmentVideoFilter) filterClass.newInstance();
+			Method filterInitMethod = filterClass.getMethod("init");
+			filterInitMethod.invoke(filter);
+			return filter;
+		} catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException
+				| SecurityException | IllegalArgumentException | InvocationTargetException | MalformedURLException e) {
+			throw new RuntimeException(
+					"jni wrapper creation failed. Bug-Mining: Ensure the wrapper was added to the javahClassNames in pom.xml. Check NativeCode.h for existing and proper signatures.",
+					e);
+		}
+	}
+
+	static URLClassLoader loader = null;
+
+	static synchronized URLClassLoader getLoader() throws MalformedURLException {
+		if (loader == null) {
+			final String RELEATIVE_TO_MAVEN_EXEC_CWD = "../../../../jniplugin/target";
+			String pluginpath = System.getProperty("pluginpath", RELEATIVE_TO_MAVEN_EXEC_CWD);
+			File myJar = new File(new File(pluginpath), "jniplugin-java-0.1.1-SNAPSHOT.jar");
+			URL[] urls = new URL[] { myJar.toURI().toURL() };
+			loader = new URLClassLoader(urls, NativeSegmentFilter.class.getClassLoader());
+		}
+		return loader;
 	}
 
 }

doc/SegmentFilters.md

@@ -153,9 +153,8 @@ Comment: The package phase (for jniplugin-java) needs to be executed twice, beca
 
 Test JNI loading within project directory:
 ```
-java -Djava.library.path=jniplugin/native/win64/target -cp jniplugin/java/target/classes  cc0.JniExample
+java -Djava.library.path=jniplugin/native/win64/target -cp jniplugin/java/target/classes  cc0.JniTest
 ```
-gi
 
 Some messages should appear: **Hello from C++!** ...
 

jniplugin/java/pom.xml

@@ -2,6 +2,13 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+  </properties>
+
     <parent>
         <groupId>de.screenflow.frankenstein</groupId>
         <artifactId>jniplugin-parent</artifactId>
@@ -15,6 +22,14 @@
     <name>Native OpenCV Filters - Java Part</name>
     <!-- Credits to http://www.tricoder.net/blog/?p=197 -->
 
+  <dependencies>
+    <dependency>
+      <groupId>de.screenflow.frankenstein</groupId>
+      <artifactId>app</artifactId>
+      <version>0.1.1-SNAPSHOT</version>
+    </dependency>
+  </dependencies>
+
    <build>
     <plugins>
       <plugin>

jniplugin/java/src/main/java/cc0/JniExample.java → jniplugin/java/src/main/java/cc0/JniTest.java

@@ -1,14 +1,14 @@
 package cc0;
 
-import de.screenflow.frankenstein.vf.jni.NativeFilter;
+import de.screenflow.frankenstein.vf.NativeFilter;
 
-public class JniExample {
-	public JniExample() throws UnsatisfiedLinkError {
+public class JniTest {
+	public JniTest() throws UnsatisfiedLinkError {
 		NativeFilter.loadLibrary();
 	}
 
 	public static void main(String[] args) {
-		new JniExample().helloNative();
+		new JniTest().helloNative();
 	}
 
 	public native void helloNative();

jniplugin/java/src/main/java/cc0/NativeExample.java

@@ -1,6 +1,6 @@
 package cc0;
 
-import de.screenflow.frankenstein.vf.jni.NativeFilter;
+import de.screenflow.frankenstein.vf.NativeFilter;
 
 public class NativeExample extends NativeFilter {
 	public NativeExample() throws UnsatisfiedLinkError {

jniplugin/java/src/main/java/de/screenflow/frankenstein/vf/jni/NativeFilter.java → jniplugin/java/src/main/java/de/screenflow/frankenstein/vf/NativeFilter.java

@@ -1,4 +1,4 @@
-package de.screenflow.frankenstein.vf.jni;
+package de.screenflow.frankenstein.vf;
 
 import java.io.File;
 import java.io.IOException;
@@ -40,7 +40,7 @@ public static void loadLibrary() throws UnsatisfiedLinkError {
         throw error; // throw again
     }
   }
-  
+
   private static void loadLibrary(String libraryName) throws UnsatisfiedLinkError {
     synchronized (loadedLibraries) {
 
@@ -67,7 +67,7 @@ private static void loadLibrary(String libraryName) throws UnsatisfiedLinkError
           Path tmpDir = Files.createTempDirectory("frankenstein");
           tmpDir.toFile().deleteOnExit();
           Path destination = tmpDir.resolve("./" + location).normalize();
-        
+
           try {
             Files.createDirectories(destination.getParent());
             Files.copy(binary, destination);
@@ -88,7 +88,7 @@ private static void loadLibrary(String libraryName) throws UnsatisfiedLinkError
           } catch (NoSuchFieldException x) {
             throw new Error("NoSuchFieldException!?", x);
           }
-        
+
           try {
             System.loadLibrary(libraryName);
           } catch (UnsatisfiedLinkError x) {
@@ -97,9 +97,9 @@ private static void loadLibrary(String libraryName) throws UnsatisfiedLinkError
                   "!!! [2] UnsatisfiedLinkError @ libraryName='" + libraryName + " message: "+x.getMessage());
              throw x;
           }
-          
+
           loadedLibraries.add(libraryName.intern());
-          
+
         } catch (final IOException x) {
           throw new Error("Error writing native library", x);
         }

jniplugin/java/src/main/java/de/screenflow/frankenstein/vf/external/ExternalSample.java

@@ -0,0 +1,11 @@
+package de.screenflow.frankenstein.vf.external;
+
+import de.screenflow.frankenstein.vf.NativeFilter;
+
+public class ExternalSample extends NativeFilter {
+	public ExternalSample() throws UnsatisfiedLinkError {
+	}
+
+	public native void init();
+	public native void process(Object mat, int frameId, Object context, Object params, Object result);
+}

jniplugin/java/src/main/java/de/screenflow/frankenstein/vf/external/ExternalSampleConfigController.java

@@ -0,0 +1,10 @@
+package de.screenflow.frankenstein.vf.external;
+
+import javafx.fxml.FXML;
+
+import de.screenflow.frankenstein.vf.segment.SegmentConfigController;
+
+
+public class ExternalSampleConfigController extends SegmentConfigController {
+
+}

jniplugin/java/src/main/java/de/screenflow/frankenstein/vf/external/ExternalSampleFilter.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2017 Oliver Rode, https://github.com/olir/Frankenstein
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package de.screenflow.frankenstein.vf.external;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Deque;
+import java.util.ArrayDeque;
+
+import org.opencv.core.Mat;
+import org.opencv.imgproc.Imgproc;
+
+import de.screenflow.frankenstein.vf.FilterContext;
+import de.screenflow.frankenstein.vf.segment.NativeSegmentFilter;
+
+public class ExternalSampleFilter extends NativeSegmentFilter<ExternalSampleConfigController> {
+
+	private final static String JNI_FILTER_CLASS = "de.screenflow.frankenstein.vf.external.ExternalSample";
+
+	private final Method jniProxyProcessMethod;
+
+	private Mat mHsvMat = new Mat();
+
+	@SuppressWarnings("unchecked")
+	public ExternalSampleFilter() throws UnsatisfiedLinkError {
+		super("externalsample", JNI_FILTER_CLASS);
+		try {
+			jniProxyProcessMethod = getJniProxyClass().getMethod("process", Object.class, int.class, Object.class);
+		} catch (Throwable e) {
+			throw new RuntimeException("jni wrapper creation failed", e);
+		}
+	}
+
+	@Override
+	public Mat process(Mat rgbaImage, int frameId, FilterContext context) {
+    Imgproc.cvtColor(rgbaImage, mHsvMat, Imgproc.COLOR_RGB2HSV_FULL);
+
+    ExternalSampleConfigController c = ((ExternalSampleConfigController)getConfigController());
+    // pass c data to proxy...
+
+    Object params = null;
+
+		try {
+			jniProxyProcessMethod.invoke(getJniProxy(), mHsvMat, frameId, context);
+		} catch (Throwable e) {
+			e.printStackTrace();
+		}
+
+		Imgproc.cvtColor(mHsvMat, rgbaImage, Imgproc.COLOR_HSV2RGB_FULL);
+
+		return rgbaImage;
+	}
+
+	@Override
+	protected void initializeController() {
+		ExternalSampleConfigController c = ((ExternalSampleConfigController)getConfigController());
+		//c.set...(...);
+		//c.initialize();
+	}
+}

jniplugin/java/src/main/java/de/screenflow/frankenstein/vf/jni/MatBlender.java

@@ -1,5 +1,7 @@
 package de.screenflow.frankenstein.vf.jni;
 
+import de.screenflow.frankenstein.vf.NativeFilter;
+
 public class MatBlender extends NativeFilter {
 	public MatBlender() throws UnsatisfiedLinkError {
 	}

jniplugin/java/src/main/java/de/screenflow/frankenstein/vf/jni/VideoEqualizer.java

@@ -1,5 +1,7 @@
 package de.screenflow.frankenstein.vf.jni;
 
+import de.screenflow.frankenstein.vf.NativeFilter;
+
 public class VideoEqualizer extends NativeFilter {
 	public VideoEqualizer() throws UnsatisfiedLinkError {
 	}