From b182cc8ec286dadbc60041d407cb83a5b5322831 Mon Sep 17 00:00:00 2001 From: Chris Gerth Date: Sun, 9 Oct 2022 10:38:39 -0500 Subject: [PATCH 1/3] Revised stream and target draw logic to divide the streams by "Raw" and "Processed" and only draw the results on the "Processed" stream --- photon-client/src/views/PipelineView.vue | 10 ++--- .../vision/pipeline/AprilTagPipeline.java | 10 ++--- .../vision/pipeline/OutputStreamPipeline.java | 41 +++++++++++++------ 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/photon-client/src/views/PipelineView.vue b/photon-client/src/views/PipelineView.vue index c1fd52a09c..911d9c6c17 100644 --- a/photon-client/src/views/PipelineView.vue +++ b/photon-client/src/views/PipelineView.vue @@ -36,7 +36,7 @@ {{ Math.round($store.state.pipelineResults.fps) }} FPS – {{ Math.min(Math.round($store.state.pipelineResults.latency), 9999) }} ms latency HSV thresholds are too broad; narrow them for better performance - stop viewing the color stream for better performance + stop viewing the raw stream for better performance - mdi-palette - Normal + mdi-import + Raw - mdi-compare - Threshold + mdi-export + Processed diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java index b6c0a0ceda..7b89a6648b 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java @@ -117,9 +117,13 @@ protected CVPipelineResult process(Frame frame, AprilTagPipelineSettings setting sumPipeNanosElapsed += rotateImageResult.nanosElapsed; } + var inputFrame = new Frame(new CVMat(rawInputMat), frameStaticProperties); + grayscalePipeResult = grayscalePipe.run(rawInputMat); sumPipeNanosElapsed += grayscalePipeResult.nanosElapsed; + var outputFrame = new Frame(new CVMat(grayscalePipeResult.output), frameStaticProperties); + List targetList; CVPipeResult> tagDetectionPipeResult; @@ -127,7 +131,6 @@ protected CVPipelineResult process(Frame frame, AprilTagPipelineSettings setting aprilTagDetectionPipe.setNativePoseEstimationEnabled(settings.solvePNPEnabled); tagDetectionPipeResult = aprilTagDetectionPipe.run(grayscalePipeResult.output); - grayscalePipeResult.output.release(); sumPipeNanosElapsed += tagDetectionPipeResult.nanosElapsed; targetList = new ArrayList<>(); @@ -150,11 +153,6 @@ protected CVPipelineResult process(Frame frame, AprilTagPipelineSettings setting var fpsResult = calculateFPSPipe.run(null); var fps = fpsResult.output; - var inputFrame = new Frame(new CVMat(rawInputMat), frameStaticProperties); - // empty output frame - var outputFrame = - Frame.emptyFrame(frameStaticProperties.imageWidth, frameStaticProperties.imageHeight); - return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, outputFrame, inputFrame); } } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java index f1547fd462..ff6351e258 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java @@ -120,42 +120,59 @@ public CVPipelineResult process( pipeProfileNanos[2] = 0; } - // Draw 2D Crosshair on input and output - var draw2dCrosshairResultOnInput = draw2dCrosshairPipe.run(Pair.of(inMat, targetsToDraw)); + // Draw 2D Crosshair on output + var draw2dCrosshairResultOnInput = draw2dCrosshairPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[3] = draw2dCrosshairResultOnInput.nanosElapsed; if (!(settings instanceof AprilTagPipelineSettings)) { + //If we're processing anything other than Apriltags... + var draw2dCrosshairResultOnOutput = draw2dCrosshairPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[4] = draw2dCrosshairResultOnOutput.nanosElapsed; - // Draw 3D Targets on input and output if necessary if (settings.solvePNPEnabled || (settings.solvePNPEnabled && settings instanceof ColoredShapePipelineSettings && ((ColoredShapePipelineSettings) settings).contourShape == ContourShape.Circle)) { - var drawOnInputResult = draw3dTargetsPipe.run(Pair.of(inMat, targetsToDraw)); - sumPipeNanosElapsed += pipeProfileNanos[7] = drawOnInputResult.nanosElapsed; + // Draw 3D Targets on input and output if possible + pipeProfileNanos[5] = 0; + pipeProfileNanos[6] = 0; + pipeProfileNanos[7] = 0; var drawOnOutputResult = draw3dTargetsPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[8] = drawOnOutputResult.nanosElapsed; } else { - pipeProfileNanos[7] = 0; - pipeProfileNanos[8] = 0; - - var draw2dTargetsOnInput = draw2dTargetsPipe.run(Pair.of(inMat, targetsToDraw)); - sumPipeNanosElapsed += pipeProfileNanos[5] = draw2dTargetsOnInput.nanosElapsed; + // Only draw 2d targets + pipeProfileNanos[5] = 0; var draw2dTargetsOnOutput = draw2dTargetsPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[6] = draw2dTargetsOnOutput.nanosElapsed; + + pipeProfileNanos[7] = 0; + pipeProfileNanos[8] = 0; } } else { + //If we are doing apriltags... if (settings.solvePNPEnabled) { - var drawOnInputResult = draw3dAprilTagsPipe.run(Pair.of(inMat, targetsToDraw)); + + //Draw 3d Apriltag markers (camera is calibrated and running in 3d mode) + pipeProfileNanos[5] = 0; + pipeProfileNanos[6] = 0; + + var drawOnInputResult = draw3dAprilTagsPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[7] = drawOnInputResult.nanosElapsed; + + pipeProfileNanos[8] = 0; + } else { - var draw2dTargetsOnInput = draw2dAprilTagsPipe.run(Pair.of(inMat, targetsToDraw)); + //Draw 2d apriltag markers + var draw2dTargetsOnInput = draw2dAprilTagsPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[5] = draw2dTargetsOnInput.nanosElapsed; + + pipeProfileNanos[6] = 0; + pipeProfileNanos[7] = 0; + pipeProfileNanos[8] = 0; } } From e1688146ef60f1be5a81b9a9e56f9980a1c17951 Mon Sep 17 00:00:00 2001 From: Chris Gerth Date: Sun, 9 Oct 2022 19:44:09 -0500 Subject: [PATCH 2/3] spotless --- .../vision/pipeline/OutputStreamPipeline.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java index ff6351e258..8c6edd55f5 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java @@ -125,7 +125,7 @@ public CVPipelineResult process( sumPipeNanosElapsed += pipeProfileNanos[3] = draw2dCrosshairResultOnInput.nanosElapsed; if (!(settings instanceof AprilTagPipelineSettings)) { - //If we're processing anything other than Apriltags... + // If we're processing anything other than Apriltags... var draw2dCrosshairResultOnOutput = draw2dCrosshairPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[4] = draw2dCrosshairResultOnOutput.nanosElapsed; @@ -153,10 +153,10 @@ public CVPipelineResult process( } } else { - //If we are doing apriltags... + // If we are doing apriltags... if (settings.solvePNPEnabled) { - - //Draw 3d Apriltag markers (camera is calibrated and running in 3d mode) + + // Draw 3d Apriltag markers (camera is calibrated and running in 3d mode) pipeProfileNanos[5] = 0; pipeProfileNanos[6] = 0; @@ -166,7 +166,7 @@ public CVPipelineResult process( pipeProfileNanos[8] = 0; } else { - //Draw 2d apriltag markers + // Draw 2d apriltag markers var draw2dTargetsOnInput = draw2dAprilTagsPipe.run(Pair.of(outMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[5] = draw2dTargetsOnInput.nanosElapsed; From 5a084995d4f7d80d2de35ec8e3954a0e4112a902 Mon Sep 17 00:00:00 2001 From: Chris Gerth Date: Sun, 9 Oct 2022 20:08:48 -0500 Subject: [PATCH 3/3] wpiformat asdfasdfasdf --- .../org/photonvision/vision/pipeline/OutputStreamPipeline.java | 1 - 1 file changed, 1 deletion(-) diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java index 8c6edd55f5..1a9eacdab8 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java @@ -155,7 +155,6 @@ public CVPipelineResult process( } else { // If we are doing apriltags... if (settings.solvePNPEnabled) { - // Draw 3d Apriltag markers (camera is calibrated and running in 3d mode) pipeProfileNanos[5] = 0; pipeProfileNanos[6] = 0;