diff --git a/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindCirclesPipe.java b/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindCirclesPipe.java index f05c79767..c79209840 100644 --- a/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindCirclesPipe.java +++ b/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindCirclesPipe.java @@ -31,12 +31,17 @@ import org.photonvision.vision.pipe.CVPipe; public class FindCirclesPipe extends CVPipe>, List, FindCirclesPipe.FindCirclePipeParams> { + // Output vector of found circles. Each vector is encoded as 3 or 4 element floating-point vector + // (x,y,radius) or (x,y,radius,votes) . private final Mat circles = new Mat(); - /** - * Runs the process for the pipe. + * Runs the process for the pipe. The reason we need a separate pipe for circles is because if we + * were to use the FindShapes pipe, we would have to assume that any shape more than 10-20+ sides + * is a circle. Only issue with such approximation is that the user would no longer be able to + * track shapes with 10-20+ sides. And hence, in order to overcome this edge case, we can use + * HoughCircles which is more flexible and accurate for finding circles. * - * @param in Input for pipe processing. + * @param in Input for pipe processing. 8-bit, single-channel, grayscale input image. * @return Result of processing. */ @Override @@ -47,27 +52,40 @@ public class FindCirclesPipe Imgproc.HoughCircles( in.getLeft(), circles, + // Detection method, see #HoughModes. The available methods are #HOUGH_GRADIENT and + // #HOUGH_GRADIENT_ALT. Imgproc.HOUGH_GRADIENT, + /*Inverse ratio of the accumulator resolution to the image resolution. + For example, if dp=1 , the accumulator has the same resolution as the input image. + If dp=2 , the accumulator has half as big width and height. For #HOUGH_GRADIENT_ALT the recommended value is dp=1.5, + unless some small very circles need to be detected. + */ 1.0, params.minDist, params.maxCannyThresh, params.accuracy, params.minRadius, params.maxRadius); + // Great, we now found the center point of the circle and it's radius, but we have no idea what + // contour it corresponds to for (int x = 0; x < circles.cols(); x++) { + // Grab the current circle we are looking at double[] c = circles.get(0, x); + // Find the center points of that circle double x_center = c[0]; double y_center = c[1]; for (Contour contour : in.getRight()) { + // Grab the moments of the current contour Moments mu = contour.getMoments(); + // Determine if the contour is within the threshold of the detected circle if (Math.abs(x_center - (mu.m10 / mu.m00)) <= params.allowableThreshold && Math.abs(y_center - (mu.m01 / mu.m00)) <= params.allowableThreshold) { + // If it is, then add it to the output array output.add(new CVShape(contour, ContourShape.Circle)); } } } - return output; } @@ -79,6 +97,17 @@ public class FindCirclesPipe private final int maxCannyThresh; private final int accuracy; + /* + * @params minDist - Minimum distance between the centers of the detected circles. + * If the parameter is too small, multiple neighbor circles may be falsely detected in addition to a true one. If it is too large, some circles may be missed. + * + * @param maxCannyThresh -First method-specific parameter. In case of #HOUGH_GRADIENT and #HOUGH_GRADIENT_ALT, it is the higher threshold of the two passed to the Canny edge detector (the lower one is twice smaller). + * Note that #HOUGH_GRADIENT_ALT uses #Scharr algorithm to compute image derivatives, so the threshold value shough normally be higher, such as 300 or normally exposed and contrasty images. + * + * + * @param allowableThreshold - When finding the corresponding contour, this is used to see how close a center should be to a contour for it to be considered THAT contour. + * Should be increased with lower resolutions and decreased with higher resolution + * */ public FindCirclePipeParams( int allowableThreshold, int minRadius, diff --git a/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindPolygonPipe.java b/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindPolygonPipe.java index 4e8c74876..b133671d6 100644 --- a/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindPolygonPipe.java +++ b/photon-server/src/main/java/org/photonvision/vision/pipe/impl/FindPolygonPipe.java @@ -49,6 +49,11 @@ public class FindPolygonPipe private CVShape getShape(Contour in) { int corners = getCorners(in); + corners = getCorners(in); + + /*The contourShape enum has predefined shapes for Circles, Triangles, and Quads + meaning any shape not fitting in those predefined shapes must be a custom shape. + */ if (ContourShape.fromSides(corners) == null) { return new CVShape(in, ContourShape.Custom); } @@ -65,18 +70,21 @@ public class FindPolygonPipe } private int getCorners(Contour contour) { + // Release previous approx approx.release(); Imgproc.approxPolyDP( contour.getMat2f(), approx, + // Converts an accuracy percentage between 1-100 to an epsilon params.accuracyPercentage / 600.0 * Imgproc.arcLength(contour.getMat2f(), true), true); + // The height of the resultant approximation is the number of vertices return (int) approx.size().height; } public static class FindPolygonPipeParams { private final double accuracyPercentage; - + // Should be a value between 0-100 public FindPolygonPipeParams(double accuracyPercentage) { this.accuracyPercentage = accuracyPercentage; } diff --git a/photon-server/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java b/photon-server/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java index 6bb8146d8..eaad2668a 100644 --- a/photon-server/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java +++ b/photon-server/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java @@ -61,7 +61,7 @@ public class ColoredShapePipeline private CVPipeResult> targetList; private final Point[] rectPoints = new Point[4]; - ColoredShapePipeline() { + public ColoredShapePipeline() { settings = new ColoredShapePipelineSettings(); } diff --git a/photon-server/src/test/java/org/photonvision/common/ShapeBenchmarkTest.java b/photon-server/src/test/java/org/photonvision/common/ShapeBenchmarkTest.java new file mode 100644 index 000000000..6e80b5c2c --- /dev/null +++ b/photon-server/src/test/java/org/photonvision/common/ShapeBenchmarkTest.java @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2020 Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.common; +/* +* Copyright (C) 2020 Photon Vision. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +*/ +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.photonvision.common.util.TestUtils; +import org.photonvision.common.util.math.MathUtils; +import org.photonvision.common.util.numbers.NumberListUtils; +import org.photonvision.vision.frame.FrameProvider; +import org.photonvision.vision.frame.provider.FileFrameProvider; +import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.opencv.ContourGroupingMode; +import org.photonvision.vision.opencv.ContourIntersectionDirection; +import org.photonvision.vision.opencv.ContourShape; +import org.photonvision.vision.pipeline.CVPipeline; +import org.photonvision.vision.pipeline.ColoredShapePipeline; +import org.photonvision.vision.pipeline.result.CVPipelineResult; + +/** Various tests that check performance on long-running tasks (i.e. a pipeline) */ +public class ShapeBenchmarkTest { + @BeforeAll + public static void init() { + TestUtils.loadLibraries(); + } + + @Test + public void Shape240pBenchmark() { + var pipeline = new ColoredShapePipeline(); + pipeline.getSettings().hsvHue.set(60, 100); + pipeline.getSettings().hsvSaturation.set(100, 255); + pipeline.getSettings().hsvValue.set(190, 255); + pipeline.getSettings().outputShowThresholded = true; + pipeline.getSettings().outputShowMultipleTargets = true; + pipeline.getSettings().contourGroupingMode = ContourGroupingMode.Single; + pipeline.getSettings().contourIntersection = ContourIntersectionDirection.Up; + pipeline.getSettings().desiredShape = ContourShape.Custom; + pipeline.getSettings().allowableThreshold = 10; + pipeline.getSettings().accuracyPercentage = 30.0; + var frameProvider = + new FileFrameProvider( + TestUtils.getWPIImagePath(TestUtils.WPI2019Image.kCargoSideStraightDark72in), + TestUtils.WPI2019Image.FOV); + + frameProvider.setImageReloading(true); + + benchmarkPipeline(frameProvider, pipeline, 5); + } + + @Test + public void Shape480pBenchmark() { + var pipeline = new ColoredShapePipeline(); + pipeline.getSettings().hsvHue.set(60, 100); + pipeline.getSettings().hsvSaturation.set(100, 255); + pipeline.getSettings().hsvValue.set(190, 255); + pipeline.getSettings().outputShowThresholded = true; + pipeline.getSettings().outputShowMultipleTargets = true; + pipeline.getSettings().contourGroupingMode = ContourGroupingMode.Single; + pipeline.getSettings().contourIntersection = ContourIntersectionDirection.Up; + pipeline.getSettings().desiredShape = ContourShape.Custom; + pipeline.getSettings().allowableThreshold = 10; + pipeline.getSettings().accuracyPercentage = 30.0; + + var frameProvider = + new FileFrameProvider( + TestUtils.getWPIImagePath(TestUtils.WPI2020Image.kBlueGoal_084in_Center), + TestUtils.WPI2020Image.FOV); + + frameProvider.setImageReloading(true); + + benchmarkPipeline(frameProvider, pipeline, 5); + } + + @Test + public void Shape720pBenchmark() { + var pipeline = new ColoredShapePipeline(); + pipeline.getSettings().hsvHue.set(60, 100); + pipeline.getSettings().hsvSaturation.set(100, 255); + pipeline.getSettings().hsvValue.set(190, 255); + pipeline.getSettings().outputShowThresholded = true; + pipeline.getSettings().outputShowMultipleTargets = true; + pipeline.getSettings().contourGroupingMode = ContourGroupingMode.Single; + pipeline.getSettings().contourIntersection = ContourIntersectionDirection.Up; + pipeline.getSettings().desiredShape = ContourShape.Custom; + pipeline.getSettings().allowableThreshold = 10; + pipeline.getSettings().accuracyPercentage = 30.0; + + var frameProvider = + new FileFrameProvider( + TestUtils.getWPIImagePath(TestUtils.WPI2020Image.kBlueGoal_084in_Center_720p), + TestUtils.WPI2020Image.FOV); + + frameProvider.setImageReloading(true); + + benchmarkPipeline(frameProvider, pipeline, 5); + } + + @Test + public void Shape1920x1440Benchmark() { + var pipeline = new ColoredShapePipeline(); + pipeline.getSettings().hsvHue.set(60, 100); + pipeline.getSettings().hsvSaturation.set(100, 255); + pipeline.getSettings().hsvValue.set(190, 255); + pipeline.getSettings().outputShowThresholded = true; + pipeline.getSettings().outputShowMultipleTargets = true; + pipeline.getSettings().contourGroupingMode = ContourGroupingMode.Single; + pipeline.getSettings().contourIntersection = ContourIntersectionDirection.Up; + pipeline.getSettings().desiredShape = ContourShape.Custom; + pipeline.getSettings().allowableThreshold = 10; + pipeline.getSettings().accuracyPercentage = 30.0; + + var frameProvider = + new FileFrameProvider( + TestUtils.getWPIImagePath(TestUtils.WPI2019Image.kCargoStraightDark72in_HighRes), + TestUtils.WPI2019Image.FOV); + + frameProvider.setImageReloading(true); + + benchmarkPipeline(frameProvider, pipeline, 5); + } + + private static

void benchmarkPipeline( + FrameProvider frameProvider, P pipeline, int secondsToRun) { + CVMat.enablePrint(false); + // warmup for 5 loops. + System.out.println("Warming up for 5 loops..."); + for (int i = 0; i < 5; i++) { + pipeline.run(frameProvider.get()); + } + + final List processingTimes = new ArrayList<>(); + final List latencyTimes = new ArrayList<>(); + + var frameProps = frameProvider.get().frameStaticProperties; + + // begin benchmark + System.out.println( + "Beginning " + + secondsToRun + + " second benchmark at resolution " + + frameProps.imageWidth + + "x" + + frameProps.imageHeight); + var benchmarkStartMillis = System.currentTimeMillis(); + do { + CVPipelineResult pipelineResult = pipeline.run(frameProvider.get()); + pipelineResult.release(); + processingTimes.add(pipelineResult.processingMillis); + latencyTimes.add(pipelineResult.getLatencyMillis()); + } while (System.currentTimeMillis() - benchmarkStartMillis < secondsToRun * 1000); + System.out.println("Benchmark complete."); + + var processingMin = Collections.min(processingTimes); + var processingMean = NumberListUtils.mean(processingTimes); + var processingMax = Collections.max(processingTimes); + + var latencyMin = Collections.min(latencyTimes); + var latencyMean = NumberListUtils.mean(latencyTimes); + var latencyMax = Collections.max(latencyTimes); + + String processingResult = + "Processing times - " + + "Min: " + + MathUtils.roundTo(processingMin, 3) + + "ms (" + + MathUtils.roundTo(1000 / processingMin, 3) + + " FPS), " + + "Mean: " + + MathUtils.roundTo(processingMean, 3) + + "ms (" + + MathUtils.roundTo(1000 / processingMean, 3) + + " FPS), " + + "Max: " + + MathUtils.roundTo(processingMax, 3) + + "ms (" + + MathUtils.roundTo(1000 / processingMax, 3) + + " FPS)"; + System.out.println(processingResult); + String latencyResult = + "Latency times - " + + "Min: " + + MathUtils.roundTo(latencyMin, 3) + + "ms (" + + MathUtils.roundTo(1000 / latencyMin, 3) + + " FPS), " + + "Mean: " + + MathUtils.roundTo(latencyMean, 3) + + "ms (" + + MathUtils.roundTo(1000 / latencyMean, 3) + + " FPS), " + + "Max: " + + MathUtils.roundTo(latencyMax, 3) + + "ms (" + + MathUtils.roundTo(1000 / latencyMax, 3) + + " FPS)"; + System.out.println(latencyResult); + } +}