Skip to content
Permalink
Browse files

First humanpose heatmap implementation

  • Loading branch information...
enrico-c committed Aug 12, 2019
1 parent 4178360 commit 064f6a25357921591cef159029ab32d5e41a4bee
@@ -82,6 +82,7 @@
protected int operationCounter = 0; // counter for ops during update. MAC should be 2 Op
protected JFrame activationsFrame = null;
protected boolean hideConvLayers = true;
protected boolean showHeatmapNotSkeleton = false;
protected boolean hideSubsamplingLayers = true;
protected JFrame kernelsFrame = null;
protected boolean printActivations = false;
@@ -308,7 +309,14 @@ public void printPerformance() {
}

public abstract void printWeights();


/**
* @param showHeatmapNotSkeleton to show heatmap instead of skeleton
*/
public void setShowHeatmapNotSkeleton(boolean showHeatmapNotSkeleton){
this.showHeatmapNotSkeleton = showHeatmapNotSkeleton;
}

/**
* @param hideConvLayers the hideConvLayers to set
*/
@@ -455,8 +463,10 @@ public void printWeights() {
//public void drawHistogram(GL2 gl, int width, int height, float lineWidth, Color color);

public float[][] getMaxActAndLocPerMap(); // to return max activation and its pixel position for each map.
public float[][] getHeatmapCNNSize(); // to return the heatmap with all joint activations.

public float[] getActivations();

}

public static void drawHistogram(GL2 gl, float[][][] activations, int width, int height, float lineWidth, Color color) {
@@ -121,7 +121,7 @@ public AbstractDavisCNNProcessor(AEChip chip) {
frameExtractor.getSupport().addPropertyChangeListener(this);
}
setEnclosedFilterChain(chain);
String deb = "5. Debug", disp = "2. Display", anal = "3. Analysis", tf = "0. Tensorflow", input = "1. Input";
String deb = "5. Debug", disp = "2. Display", anal = "4. Analysis", tf = "0. Tensorflow", input = "1. Input";
setPropertyTooltip("loadNetwork", "Load an XML or PB file containing a CNN");
//setPropertyTooltip("loadLabels", "Load labels for output units");
//setPropertyTooltip(disp, "showOutputAsBarChart", "displays activity of output units as bar chart, where height indicates activation");
@@ -69,6 +69,8 @@
private ArrayList<String> ioLayers = new ArrayList();
SavedModelBundle savedModelBundle = null;
private ImageDisplay imageDisplay;

private boolean showHeatmapNotSkeletonFlag;

public DavisCNNTensorFlow(AbstractDavisCNNProcessor processor) {
super(processor);
@@ -441,14 +443,18 @@ public void printWeights() {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}

@Override
public void setShowHeatmapNotSkeleton(boolean showHeatmapNotSkeleton) {
showHeatmapNotSkeletonFlag = showHeatmapNotSkeleton;
}


float[] outActivations;
int numUnits, height, width, chans, nPixPerRow;
float[][] tmpMaxActAndLocPerMap = null;
float[][] tmpMaxActAndLocPerMap = null;

float[][] maxActAndLocPerMap; // stores max activation and location for each map.
//float[][] sumHeatmapsOutputSize; // variable to store sum of heatmaps, with CNN output size. To be upsampled for plotting.
float[][] heatmapCNNSize; // to store sum of heatmaps, with CNN output size. To be upsampled for plotting.

public class OutputLayer implements AbstractDavisCNN.OutputLayer {

@@ -472,7 +478,7 @@ public OutputLayer(float[] output) {
numUnits = height * width * chans; // check if needed or can be removed.
nPixPerRow = chans * width;
tmpMaxActAndLocPerMap = new float[chans][3]; // hardcoded, max value and x,y position.
//sumHeatmapsOutputSize = new float[height][width];
heatmapCNNSize = new float[height][width];
log.info(String.format("*** DONE Instantiating problem-specific constants: [height, width, chans], nPixPerRow, size tmpMaxActAndLocPerMap, size heatmap."));
}

@@ -485,44 +491,32 @@ public OutputLayer(float[] output) {
for (int c = 0; c < chans; c++) {
final int idx=c+x*chans+y*nPixPerRow;
final float act=outActivations[idx];
// initialize the value of max for each CNN output.
if ((y==0) && (x==0)) {
tmpMaxActAndLocPerMap[c][0]=act;
tmpMaxActAndLocPerMap[c][1]=0;
tmpMaxActAndLocPerMap[c][2]=0;

// to display heatmap
if (showHeatmapNotSkeletonFlag){
// only instantaneous heatmap is displayed
if (c==0){ heatmapCNNSize[y][x]=act;} //to avoid initializing the map to zero.
else{ heatmapCNNSize[y][x] += act; }
}
else{
if(act>tmpMaxActAndLocPerMap[c][0]){
//order is: [mapIdx][activation y x]
else{ // to display skeleton
// initialize the value of max for each CNN output.
if ((y==0) && (x==0)) {
tmpMaxActAndLocPerMap[c][0]=act;
tmpMaxActAndLocPerMap[c][1]=y;
tmpMaxActAndLocPerMap[c][2]=x;
tmpMaxActAndLocPerMap[c][1]=0;
tmpMaxActAndLocPerMap[c][2]=0;
}
else{
if(act>tmpMaxActAndLocPerMap[c][0]){
//order is: [mapIdx][activation y x]
tmpMaxActAndLocPerMap[c][0]=act;
tmpMaxActAndLocPerMap[c][1]=y;
tmpMaxActAndLocPerMap[c][2]=x;
}
}
}
}
}

}
// int mapIdx, xIdx, yIdx;
// for (int i = 0; i < outActivations.length; i++) { // loop over the whole 1d array
// mapIdx = i % outShape[2];
// xIdx = (i / outShape[2]) % outShape[1];
// yIdx = ((i / outShape[2]) / outShape[1]) % outShape[0];
//
// //order is: [mapIdx][activation y x]
// // write the first nHeatmaps elements
// if (i < outShape[2]) {
// tmpMaxActAndLocPerMap[mapIdx][1] = yIdx;
// tmpMaxActAndLocPerMap[mapIdx][2] = xIdx;
// tmpMaxActAndLocPerMap[mapIdx][0] = outActivations[i];
// } else { // otherwise replace only if a larger value is found.
// if (tmpMaxActAndLocPerMap[mapIdx][0] < outActivations[i]) {
// tmpMaxActAndLocPerMap[mapIdx][1] = yIdx;
// tmpMaxActAndLocPerMap[mapIdx][2] = xIdx;
// tmpMaxActAndLocPerMap[mapIdx][0] = outActivations[i];
// }
// }
// }

//long t1 = System.nanoTime();
maxActAndLocPerMap = tmpMaxActAndLocPerMap;
@@ -537,8 +531,6 @@ public int getNumUnits() {
return numUnits;
}



//@Override
//public int[] getMaxActivatedUnit() {
// return maxActivatedUnitPerMap;
@@ -564,12 +556,17 @@ public int getNumUnits() {
public float[][] getMaxActAndLocPerMap(){
return maxActAndLocPerMap;
}

@Override
public float[][] getHeatmapCNNSize(){
return heatmapCNNSize;
}

@Override
public float[] getActivations() {
return outActivations;
}

}

public class InputLayer implements AbstractDavisCNN.InputLayer {

0 comments on commit 064f6a2

Please sign in to comment.
You can’t perform that action at this time.