From e48f96420e2a8ea0bb564bc3a0568ef42ef47b08 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 1 Dec 2019 13:29:30 -0500 Subject: [PATCH] Modify pipes to pass around same Mat, fix Streamer leak --- .../chameleonvision/vision/VisionProcess.java | 9 +++++-- .../vision/camera/USBCameraCapture.java | 5 +++- .../vision/pipeline/CVPipeline2d.java | 12 ++++----- .../vision/pipeline/DriverVisionPipeline.java | 4 +-- .../vision/pipeline/pipes/BlurPipe.java | 26 +++++++++---------- .../pipeline/pipes/Draw2dContoursPipe.java | 20 ++++++++------ .../pipeline/pipes/ErodeDilatePipe.java | 14 +++++----- .../vision/pipeline/pipes/HsvPipe.java | 9 +++---- 8 files changed, 54 insertions(+), 45 deletions(-) diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java b/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java index 222723ebc..2f841930f 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -246,6 +246,7 @@ public class VisionProcess { Mat camFrame = camData.getLeft(); if (camFrame.cols() > 0 && camFrame.rows() > 0) { CVPipelineResult result = pipelineManager.getCurrentPipeline().runPipeline(camFrame); + camFrame.release(); if (result != null) { result.setTimestamp(camData.getRight()); @@ -258,7 +259,6 @@ public class VisionProcess { try { streamFrameQueue.clear(); streamFrameQueue.add(lastPipelineResult.outputMat); - camFrame.release(); } catch (Exception e) { Debug.printInfo("Vision running faster than stream."); } @@ -295,7 +295,12 @@ public class VisionProcess { protected void process() { if (!streamFrameQueue.isEmpty()) { try { - streamer.runStream(streamFrameQueue.take()); + Mat tempMat = new Mat(); + Mat tempMat2 = streamFrameQueue.take(); + tempMat2.copyTo(tempMat); + tempMat2.release(); + streamer.runStream(tempMat); + tempMat.release(); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java index 0cba6a2dd..f8b229c6d 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java @@ -40,7 +40,10 @@ public class USBCameraCapture implements CameraCapture { public Pair getFrame() { Long deltaTime; // TODO: Why multiply by 1000 here? - deltaTime = cvSink.grabFrame(imageBuffer) * 1000L; + Mat tempMat = new Mat(); + deltaTime = cvSink.grabFrame(tempMat) * 1000L; + tempMat.copyTo(imageBuffer); + tempMat.release(); return Pair.of(imageBuffer, deltaTime); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index c3f113b04..dd211a5a1 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -79,7 +79,7 @@ public class CVPipeline2d extends CVPipeline rotateFlipResult = rotateFlipPipe.run(inputMat); totalPipelineTimeNanos += rotateFlipResult.getRight(); - Pair blurResult = blurPipe.run(rotateFlipResult.getLeft()); - totalPipelineTimeNanos += blurResult.getRight(); +// Pair blurResult = blurPipe.run(rotateFlipResult.getLeft()); +// totalPipelineTimeNanos += blurResult.getRight(); - Pair erodeDilateResult = erodeDilatePipe.run(blurResult.getLeft()); + Pair erodeDilateResult = erodeDilatePipe.run(rotateFlipResult.getLeft()); totalPipelineTimeNanos += erodeDilateResult.getRight(); Pair hsvResult = hsvPipe.run(erodeDilateResult.getLeft()); @@ -163,7 +163,7 @@ public class CVPipeline2d extends CVPipeline blankList = List.of(); - private final MemoryManager memoryManager = new MemoryManager(50); + private final MemoryManager memoryManager = new MemoryManager(200, 20000); public DriverVisionPipeline(CVPipelineSettings settings) { super(settings); @@ -46,7 +46,7 @@ public class DriverVisionPipeline extends CVPipeline rotateFlipResult = rotateFlipPipe.run(inputMat); Pair draw2dContoursResult = draw2dContoursPipe.run(Pair.of(rotateFlipResult.getLeft(), blankList)); - memoryManager.run(Main.testMode); + memoryManager.run(); return new DriverPipelineResult(null, draw2dContoursResult.getLeft(), 0); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java index eeb175356..a6ee819b5 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java @@ -25,20 +25,20 @@ public class BlurPipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - if (blurSize > 0) { - input.copyTo(processBuffer); - try { - Imgproc.blur(processBuffer, processBuffer, new Size(blurSize, blurSize)); - processBuffer.copyTo(outputMat); - processBuffer.release(); - } catch (CvException e) { - System.err.println("(BlurPipe) Exception thrown by OpenCV: \n" + e.getMessage()); - } - } else { - input.copyTo(outputMat); - } +// if (blurSize > 0) { +// input.copyTo(processBuffer); +// try { +// Imgproc.blur(processBuffer, processBuffer, new Size(blurSize, blurSize)); +// processBuffer.copyTo(outputMat); +// processBuffer.release(); +// } catch (CvException e) { +// System.err.println("(BlurPipe) Exception thrown by OpenCV: \n" + e.getMessage()); +// } +// } else { +// input.copyTo(outputMat); +// } long processTime = System.nanoTime() - processStartNanos; - return Pair.of(outputMat, processTime); + return Pair.of(input, processTime); } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java index af2a1489a..af7655af6 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java @@ -21,6 +21,7 @@ public class Draw2dContoursPipe implements Pipe>, Ma private Point[] vertices = new Point[4]; private List drawnContours = new ArrayList<>(); + private MatOfPoint contour = new MatOfPoint(); @SuppressWarnings("FieldCanBeLocal") private Point xMax = new Point(), xMin = new Point(), yMax = new Point(), yMin = new Point(); @@ -41,7 +42,7 @@ public class Draw2dContoursPipe implements Pipe>, Ma if (settings.showCrosshair || settings.showCentroid || settings.showMaximumBox || settings.showRotatedBox) { // input.getLeft().copyTo(processBuffer); - processBuffer = input.getLeft(); +// processBuffer = input.getLeft(); if (input.getRight().size() > 0) { for (int i = 0; i < input.getRight().size(); i++) { @@ -55,21 +56,24 @@ public class Draw2dContoursPipe implements Pipe>, Ma drawnContours.clear(); r.points(vertices); - MatOfPoint contour = new MatOfPoint(vertices); + contour.fromArray(vertices); +// MatOfPoint contour = new MatOfPoint(vertices); drawnContours.add(contour); if (settings.showCentroid) { - Imgproc.circle(processBuffer, r.center, 3, Helpers.colorToScalar(settings.centroidColor)); + Imgproc.circle(input.getLeft(), r.center, 3, Helpers.colorToScalar(settings.centroidColor)); } if (settings.showRotatedBox) { - Imgproc.drawContours(processBuffer, drawnContours, 0, Helpers.colorToScalar(settings.rotatedBoxColor), settings.boxOutlineSize); + Imgproc.drawContours(input.getLeft(), drawnContours, 0, Helpers.colorToScalar(settings.rotatedBoxColor), settings.boxOutlineSize); } if (settings.showMaximumBox) { Rect box = Imgproc.boundingRect(contour); - Imgproc.rectangle(processBuffer, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), Helpers.colorToScalar(settings.maximumBoxColor), settings.boxOutlineSize); + Imgproc.rectangle(input.getLeft(), new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), Helpers.colorToScalar(settings.maximumBoxColor), settings.boxOutlineSize); } + +// contour.release(); } } @@ -78,8 +82,8 @@ public class Draw2dContoursPipe implements Pipe>, Ma xMin.set(new double[] {camProps.centerX - 10, camProps.centerY}); yMax.set(new double[] {camProps.centerX, camProps.centerY + 10}); yMin.set(new double[] {camProps.centerX, camProps.centerY - 10}); - Imgproc.line(processBuffer, xMax, xMin, Helpers.colorToScalar(settings.crosshairColor), 2); - Imgproc.line(processBuffer, yMax, yMin, Helpers.colorToScalar(settings.crosshairColor), 2); + Imgproc.line(input.getLeft(), xMax, xMin, Helpers.colorToScalar(settings.crosshairColor), 2); + Imgproc.line(input.getLeft(), yMax, yMin, Helpers.colorToScalar(settings.crosshairColor), 2); } // processBuffer.copyTo(outputMat); @@ -89,7 +93,7 @@ public class Draw2dContoursPipe implements Pipe>, Ma } long processTime = System.nanoTime() - processStartNanos; - return Pair.of(processBuffer, processTime); + return Pair.of(input.getLeft(), processTime); } public static class Draw2dContoursSettings { diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java index 215e3ecdd..5b8a67319 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java @@ -31,23 +31,23 @@ public class ErodeDilatePipe implements Pipe { long processStartNanos = System.nanoTime(); if (erode || dilate) { - input.copyTo(processBuffer); +// input.copyTo(processBuffer); if (erode) { - Imgproc.erode(processBuffer, processBuffer, kernel); + Imgproc.erode(input, input, kernel); } if (dilate) { - Imgproc.dilate(processBuffer, processBuffer, kernel); + Imgproc.dilate(input, input, kernel); } - processBuffer.copyTo(outputMat); - processBuffer.release(); +// processBuffer.copyTo(outputMat); +// processBuffer.release(); } else { - input.copyTo(outputMat); +// input.copyTo(outputMat); } long processTime = System.nanoTime() - processStartNanos; - return Pair.of(outputMat, processTime); + return Pair.of(input, processTime); } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java index 9a271e8bd..5a1ea0eec 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java @@ -29,18 +29,15 @@ public class HsvPipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - input.copyTo(processBuffer); + input.copyTo(outputMat); try { - Imgproc.cvtColor(processBuffer, processBuffer, Imgproc.COLOR_BGR2HSV, 3); - Core.inRange(processBuffer, hsvLower, hsvUpper, processBuffer); + Imgproc.cvtColor(outputMat, outputMat, Imgproc.COLOR_BGR2HSV, 3); + Core.inRange(outputMat, hsvLower, hsvUpper, outputMat); } catch (CvException e) { System.err.println("(HsvPipe) Exception thrown by OpenCV: \n" + e.getMessage()); } - processBuffer.copyTo(outputMat); - processBuffer.release(); - long processTime = System.nanoTime() - processStartNanos; return Pair.of(outputMat, processTime); }