Skip to content

Commit

Permalink
Added GT import for npz file (MVSEC dataset).
Browse files Browse the repository at this point in the history
  • Loading branch information
@wzygzlm
wzygzlm committed May 26, 2020
1 parent af34f6c commit 68e5ff4
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 15 deletions.
3 changes: 3 additions & 0 deletions ivy.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,9 @@ MA 02110-1301 USA
<!--used in telluride for communicating to foosball-->
<dependency org="redis.clients" name="jedis" rev="2.9.0" />

<!-- used for numpy data file format npy and npz support in JVM -->
<dependency org="org.jetbrains.bio" name="npy" rev="0.3.3"/>

</dependencies>
</ivy-module>

1 change: 1 addition & 0 deletions ivysettings.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
<chain name="default" returnFirst="true" checkmodified="true">
<resolver ref="local"/>
<resolver ref="main"/>
<ibiblio name="spring_lib_m" root="https://repo.spring.io/libs-release/" m2compatible="true" />
<ibiblio name="rosjava_mvn_repo" m2compatible="true"
root="https://github.com/rosjava/rosjava_mvn_repo" />
</chain>
Expand Down
19 changes: 10 additions & 9 deletions src/ch/unizh/ini/jaer/projects/minliu/PatchMatchFlow.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,8 @@ public enum SliceMethod {

int outerCircle[][] = circle2;
int outerCircleSize = outerCircle.length;
int xOuterOffset[] = new int[outerCircleSize];
int yOuterOffset[] = new int[outerCircleSize];
int xOuterOffset[] = new int[outerCircleSize];
int outerTsValue[] = new int[outerCircleSize];

private HWCornerPointRenderer keypointFilter = null;
Expand Down Expand Up @@ -431,7 +431,7 @@ synchronized public EventPacket filterPacket(EventPacket in) {
// continue;
// }

if(ein.timestamp == 80588764)
if(ein.timestamp == 151322924)
{
// ein.y = (short)(ein.y + 4);
int tmp = 1;
Expand Down Expand Up @@ -541,7 +541,7 @@ synchronized public EventPacket filterPacket(EventPacket in) {
if(Math.abs(result.dy) >= 4)
{
int tmp = 0;
}
}
if(ein.timestamp == 80059493)
{
int tmp = 1;
Expand Down Expand Up @@ -3176,7 +3176,7 @@ boolean PatchFastDetectorisFeature(PolarityEvent ein) {
final int innerSize = circle3_.length;
final int outerSize = circle4_.length;

int innerStartX, innerEndX, innerStartY, innerEndY;
int innerStartX = 0, innerEndX = 0, innerStartY = 0, innerEndY = 0;
int outerStartX, outerEndX, outerStartY, outerEndY;

// only check if not too close to border
Expand Down Expand Up @@ -3291,11 +3291,12 @@ boolean PatchFastDetectorisFeature(PolarityEvent ein) {
// if (found_streak)
{
found_streak_outer = false;
boolean exit_loop = false;

FastDetectorisFeature_label6:
for (int i = 0; i < outerSize; i++) {
for (int streak_size = outerCircleSize - 1; streak_size >= 3; streak_size--) {
FastDetectorisFeature_label5:
for (int streak_size = outerCircleSize - 1; streak_size >= 3; streak_size--) {
for (int i = 0; i < outerSize; i++) {
// check that first event is larger than neighbor
if ( Math.abs(featureSlice[pix_x + circle4_[i][0]][pix_y + circle4_[i][1]] - centerValue) < Math.abs(featureSlice[pix_x + circle4_[(i - 1 + outerSize) % outerSize][0]][pix_y + circle4_[(i - 1 + outerSize) % outerSize][1]] - centerValue)) {
continue;
Expand Down Expand Up @@ -3361,15 +3362,15 @@ boolean PatchFastDetectorisFeature(PolarityEvent ein) {
if(min_t - max_t != 15 || streak_size != 10)
{
// found_streak_outer = false;
}
}
exit_loop = true;
break;
}
}
if (found_streak_outer) {
if (found_streak_outer || exit_loop) {
break;
}
}

}

switch(cornerCircleSelection){
Expand Down
80 changes: 74 additions & 6 deletions src/ch/unizh/ini/jaer/projects/rbodo/opticalflow/AbstractMotionFlowIMU.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@
import java.beans.PropertyChangeListener;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Observable;
import java.util.Observer;
import java.util.logging.Level;
Expand Down Expand Up @@ -46,6 +48,9 @@
import net.sf.jaer.util.WarningDialogWithDontShowPreference;
import net.sf.jaer.util.filter.LowpassFilter3D;
import net.sf.jaer.util.filter.LowpassFilter3D.Point3D;
import org.jetbrains.bio.npy.NpyArray;
import org.jetbrains.bio.npy.NpzEntry;
import org.jetbrains.bio.npy.NpzFile;

/**
* Abstract base class for motion flow filters. The filters that extend this
Expand Down Expand Up @@ -156,6 +161,7 @@ abstract public class AbstractMotionFlowIMU extends EventFilter2D implements Obs
double[][] vxGTframe, vyGTframe, tsGTframe;
public float vxGT, vyGT, vGT;
private boolean importedGTfromMatlab;
private boolean importedGTfromNPZ = false;

// Discard events that are considerably faster than average
private float avgSpeed = 0;
Expand Down Expand Up @@ -211,7 +217,11 @@ abstract public class AbstractMotionFlowIMU extends EventFilter2D implements Obs
protected SingleCameraCalibration cameraCalibration = null;
int uidx;
protected boolean useColorForMotionVectors = getBoolean("useColorForMotionVectors", true);

private int[] tsShape;
private float[] tsData;
private float[] xOFData;
private float[] yOFData;

public AbstractMotionFlowIMU(AEChip chip) {
super(chip);
addObservers(chip);
Expand Down Expand Up @@ -273,6 +283,7 @@ public AbstractMotionFlowIMU(AEChip chip) {
setPropertyTooltip(imuTT, "startIMUCalibration", "<html> Starts estimating the IMU offsets based on next calibrationSamples samples. Should be used only with stationary recording to store these offsets in the preferences. <p> <b>measureAccuracy</b> must be selected as well to actually do the calibration.");
setPropertyTooltip(imuTT, "eraseIMUCalibration", "Erases the IMU offsets to zero. Can be used to observe effect of these offsets on a stationary recording in the IMUFlow filter.");
setPropertyTooltip(imuTT, "importGTfromMatlab", "Allows importing two 2D-arrays containing the x-/y- components of the motion flow field used as ground truth.");
setPropertyTooltip(imuTT, "importGTfromNPZ", "Allows importing ground truth from numpy file, only used for MVSEC dataset.");
setPropertyTooltip(imuTT, "resetGroundTruth", "Resets the ground truth optical flow that was imported from matlab. Used in the measureAccuracy option.");
setPropertyTooltip(imuTT, "selectLoggingFolder", "Allows selection of the folder to store the measured accuracies and optical flow events.");
// setPropertyTooltip(motionFieldTT, "motionFieldMixingFactor", "Flow events are mixed with the motion field with this factor. Use 1 to replace field content with each event, or e.g. 0.01 to update only by 1%.");
Expand Down Expand Up @@ -325,7 +336,6 @@ public final void maybeAddListeners(AEChip chip) {
}
if (!addTimeStampsResetPropertyChangeListener) {
chip.getAeViewer().addPropertyChangeListener(AEViewer.EVENT_TIMESTAMPS_RESET, this);
chip.getAeViewer().getSupport().addPropertyChangeListener(AEInputStream.EVENT_REWOUND, this);
addTimeStampsResetPropertyChangeListener = true;
}
}
Expand All @@ -339,9 +349,9 @@ synchronized public void doImportGTfromMatlab() {
chooser.setFileSelectionMode(JFileChooser.FILES_ONLY);
chooser.setMultiSelectionEnabled(false);
if (chooser.showOpenDialog(chip.getAeViewer().getFilterFrame()) == JFileChooser.APPROVE_OPTION) {
try {
try {
vxGTframe = ((MLDouble) (new MatFileReader(chooser.getSelectedFile().getPath())).getMLArray("vxGT")).getArray();
vyGTframe = ((MLDouble) (new MatFileReader(chooser.getSelectedFile().getPath())).getMLArray("vyGT")).getArray();
vyGTframe = ((MLDouble) (new MatFileReader(chooser.getSelectedFile().getPath())).getMLArray("vyGT")).getArray();
tsGTframe = ((MLDouble) (new MatFileReader(chooser.getSelectedFile().getPath())).getMLArray("ts")).getArray();
importedGTfromMatlab = true;
log.info("Imported ground truth file");
Expand All @@ -351,6 +361,50 @@ synchronized public void doImportGTfromMatlab() {
}
}


// Allows importing two 2D-arrays containing the x-/y- components of the
// motion flow field used as ground truth from numpy file .npz.
// Primarily used for MVSEC dataset.
synchronized public void doImportGTfromNPZ() {
JFileChooser chooser = new JFileChooser();
chooser.setDialogTitle("Choose ground truth file");
chooser.setFileSelectionMode(JFileChooser.FILES_ONLY);
chooser.setMultiSelectionEnabled(false);
if (chooser.showOpenDialog(chip.getAeViewer().getFilterFrame()) == JFileChooser.APPROVE_OPTION) {
String fileName = chooser.getSelectedFile().getPath();
Path filePath = new File(fileName).toPath();
NpzFile.Reader reader = NpzFile.read(filePath);
List<NpzEntry> entryList = reader.introspect();
entryList.size();
NpyArray tsArray = reader.get("timestamps", 1 << 18);
tsData = tsArray.asFloatArray();
tsShape = tsArray.getShape();
float firtTs = tsData[0];
for(int i = 0; i < tsShape[0]; i++)
{
tsData[i] = tsData[i] - firtTs;
tsData[i] = tsData[i] * 1e6f; // Convert to us.
}
NpyArray xOFArray = reader.get("x_flow_dist", 1 << 18);
xOFData = xOFArray.asFloatArray();
for (int i = 0; i < xOFData.length; i++)
{
xOFData[i] = (float)(xOFData[i]/0.05);
}
int[] xOFShape = xOFArray.getShape();
NpyArray yOFArray = reader.get("y_flow_dist", 1 << 18);
yOFData = yOFArray.asFloatArray();
for (int i = 0; i < yOFData.length; i++)
{
yOFData[i] = (float)(yOFData[i]/0.05);
}
int[] yOFShape = yOFArray.getShape();
reader.close();
importedGTfromNPZ = true;
log.info("Imported ground truth file from npz file.");
}
}

// Allows exporting flow vectors that were accumulated between tmin and tmax
// to a mat-file which can be processed in MATLAB.
public void exportFlowToMatlab(final int tmin, final int tmax) {
Expand Down Expand Up @@ -529,10 +583,25 @@ public boolean calculateImuFlow(Object o) {
if (ts >= tsGTframe[0][0] && ts < tsGTframe[0][1]) {
vx = (float) vxGTframe[y][x];
vy = (float) vyGTframe[y][x];
v = (float) Math.sqrt(vx * vx + vy * vy);
} else {
vx = 0;
vy = 0;
}
} else if (importedGTfromNPZ) {
int frameIdx = ts/50000; // MVSEC's OF is updated every 50ms.
if(frameIdx < tsShape[0])
{
vx = (float)xOFData[frameIdx * (260 * 346) + y * 346 + x];
vy = (float)yOFData[frameIdx * (260 * 346) + y * 346 + x];
v = (float) Math.sqrt(vx * vx + vy * vy);
}
else
{
vx = 0;
vy = 0;
v = 0;
}
} else {
// otherwise, if not IMU sample, use last IMU sample to compute GT flow from last IMU sample and event location
// then transform, then move them back to their origin.
Expand Down Expand Up @@ -917,7 +986,6 @@ public void annotate(GLAutoDrawable drawable) {
gl.glRasterPos2i(chip.getSizeX() / 2, offset);
chip.getCanvas().getGlut().glutBitmapString(GLUT.BITMAP_HELVETICA_18,
motionFlowStatistics.endpointErrorAbs.graphicsString("AEE(abs):", "pps"));
motionFlowStatistics.endpointErrorAbs.clear();
gl.glPopMatrix();
gl.glPushMatrix();
gl.glRasterPos2i(chip.getSizeX() / 2, 2 * offset);
Expand Down Expand Up @@ -1149,7 +1217,7 @@ synchronized public void doStopLoggingMotionVectorEvents() {
motionVectorEventLogger.setEnabled(false);
motionVectorEventLogger = null;
}

protected void logMotionVectorEvents(EventPacket ep) {
if (motionVectorEventLogger == null) {
return;
Expand Down

0 comments on commit 68e5ff4

Please sign in to comment.