From c683bebf7692848fb1350821c50916888cef5b8e Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Fri, 12 Jun 2020 19:58:58 -0400 Subject: [PATCH] Tests now print when run, added BenchmarkTests, CVMat togglable output --- chameleon-server/build.gradle | 3 + .../common/util/numbers/NumberListUtils.java | 98 +++++++++++ .../frame/provider/FileFrameProvider.java | 9 +- .../common/vision/opencv/CVMat.java | 8 +- .../chameleonvision/common/BenchmarkTest.java | 155 ++++++++++++++++++ 5 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 chameleon-server/src/main/java/com/chameleonvision/common/util/numbers/NumberListUtils.java create mode 100644 chameleon-server/src/test/java/com/chameleonvision/common/BenchmarkTest.java diff --git a/chameleon-server/build.gradle b/chameleon-server/build.gradle index 03f0070f2..7f5047736 100644 --- a/chameleon-server/build.gradle +++ b/chameleon-server/build.gradle @@ -111,6 +111,9 @@ sourceSets { test { useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed", "standardOut", "standardError" + } } task testHeadless(type: Test) { diff --git a/chameleon-server/src/main/java/com/chameleonvision/common/util/numbers/NumberListUtils.java b/chameleon-server/src/main/java/com/chameleonvision/common/util/numbers/NumberListUtils.java new file mode 100644 index 000000000..d66e62723 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/common/util/numbers/NumberListUtils.java @@ -0,0 +1,98 @@ +package com.chameleonvision.common.util.numbers; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.StringJoiner; + +@SuppressWarnings("unused") +public class NumberListUtils { + + /** + * @param collection an ArrayList of Comparable objects + * @return the median of collection + */ + public static double median(List collection, Comparator comp) { + double result; + int n = collection.size() / 2; + + if (collection.size() % 2 == 0) // even number of items; find the middle two and average them + result = + (nthSmallest(collection, n - 1, comp).doubleValue() + + nthSmallest(collection, n, comp).doubleValue()) + / 2.0; + else // odd number of items; return the one in the middle + result = nthSmallest(collection, n, comp).doubleValue(); + + return result; + } + + public static String toString(List collection) { + return toString(collection, ""); + } + + public static String toString(List collection, String suffix) { + StringJoiner joiner = new StringJoiner(", "); + for (T x : collection) { + String s = x.doubleValue() + suffix; + joiner.add(s); + } + return joiner.toString(); + } + + /** + * @param collection an ArrayList of Numbers + * @return the mean of collection + */ + public static double mean(final List collection) { + BigDecimal sum = BigDecimal.ZERO; + for (final Number number : collection) { + sum = sum.add(BigDecimal.valueOf(number.doubleValue())); + } + return (sum.doubleValue() / collection.size()); + } + + /** + * @param collection a collection of Comparable objects + * @param n the position of the desired object, using the ordering defined on the collection + * elements + * @return the nth smallest object + */ + public static T nthSmallest(List collection, int n, Comparator comp) { + T result, pivot; + ArrayList underPivot = new ArrayList<>(), + overPivot = new ArrayList<>(), + equalPivot = new ArrayList<>(); + + // choosing a pivot is a whole topic in itself. + // this implementation uses the simple strategy of grabbing something from the middle of the + // ArrayList. + + pivot = collection.get(n / 2); + + // split collection into 3 lists based on comparison with the pivot + + for (T obj : collection) { + int order = comp.compare(obj, pivot); + + if (order < 0) // obj < pivot + underPivot.add(obj); + else if (order > 0) // obj > pivot + overPivot.add(obj); + else // obj = pivot + equalPivot.add(obj); + } // for each obj in collection + + // recurse on the appropriate collection + + if (n < underPivot.size()) result = nthSmallest(underPivot, n, comp); + else if (n < underPivot.size() + equalPivot.size()) // equal to pivot; just return it + result = pivot; + else // everything in underPivot and equalPivot is too small. Adjust n accordingly in the + // recursion. + result = nthSmallest(overPivot, n - underPivot.size() - equalPivot.size(), comp); + + return result; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/common/vision/frame/provider/FileFrameProvider.java b/chameleon-server/src/main/java/com/chameleonvision/common/vision/frame/provider/FileFrameProvider.java index 83e74bdb8..24de9c035 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/common/vision/frame/provider/FileFrameProvider.java +++ b/chameleon-server/src/main/java/com/chameleonvision/common/vision/frame/provider/FileFrameProvider.java @@ -18,10 +18,9 @@ public class FileFrameProvider implements FrameProvider { private static int count = 0; private Frame m_frame; - private Path m_path; + private final Path m_path; - private double m_fov; - private FrameStaticProperties m_properties; + private final double m_fov; private boolean m_reloadImage; @@ -54,7 +53,7 @@ public class FileFrameProvider implements FrameProvider { Mat image = Imgcodecs.imread(m_path.toString()); if (image.cols() > 0 && image.rows() > 0) { - m_properties = new FrameStaticProperties(image.width(), image.height(), m_fov); + FrameStaticProperties m_properties = new FrameStaticProperties(image.width(), image.height(), m_fov); m_frame = new Frame(new CVMat(image), m_properties); } else { throw new RuntimeException("Image loading failed!"); @@ -83,6 +82,8 @@ public class FileFrameProvider implements FrameProvider { @Override public Frame get() { if (m_reloadImage) { + if (m_frame != null) m_frame.release(); + m_frame = null; loadImage(); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/common/vision/opencv/CVMat.java b/chameleon-server/src/main/java/com/chameleonvision/common/vision/opencv/CVMat.java index 9c713a55b..74f86aac6 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/common/vision/opencv/CVMat.java +++ b/chameleon-server/src/main/java/com/chameleonvision/common/vision/opencv/CVMat.java @@ -7,6 +7,8 @@ import org.opencv.core.Mat; public class CVMat implements Releasable { private static final HashSet allMats = new HashSet<>(); + private static boolean shouldPrint; + private final Mat mat; public CVMat() { @@ -23,7 +25,7 @@ public class CVMat implements Releasable { public CVMat(Mat mat) { this.mat = mat; - if (allMats.add(mat)) { + if (allMats.add(mat) && shouldPrint) { System.out.println( "(CVMat) Added new Mat (count: " + allMats.size() @@ -45,4 +47,8 @@ public class CVMat implements Releasable { public static int getMatCount() { return allMats.size(); } + + public static void enablePrint(boolean enabled) { + shouldPrint = enabled; + } } diff --git a/chameleon-server/src/test/java/com/chameleonvision/common/BenchmarkTest.java b/chameleon-server/src/test/java/com/chameleonvision/common/BenchmarkTest.java new file mode 100644 index 000000000..ad366003f --- /dev/null +++ b/chameleon-server/src/test/java/com/chameleonvision/common/BenchmarkTest.java @@ -0,0 +1,155 @@ +package com.chameleonvision.common; + +import com.chameleonvision.common.util.TestUtils; +import com.chameleonvision.common.util.math.MathUtils; +import com.chameleonvision.common.util.numbers.NumberListUtils; +import com.chameleonvision.common.vision.frame.FrameProvider; +import com.chameleonvision.common.vision.frame.provider.FileFrameProvider; +import com.chameleonvision.common.vision.opencv.CVMat; +import com.chameleonvision.common.vision.opencv.ContourGroupingMode; +import com.chameleonvision.common.vision.opencv.ContourIntersectionDirection; +import com.chameleonvision.common.vision.pipeline.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** Various tests that check performance on long-running tasks (i.e. a pipeline) */ +public class BenchmarkTest { + @BeforeAll + public static void init() { + TestUtils.loadLibraries(); + } + + @Test + public void Reflective240pBenchmark() { + var pipeline = new ReflectivePipeline(); + pipeline.getSettings().hsvHue.set(60, 100); + pipeline.getSettings().hsvSaturation.set(100, 255); + pipeline.getSettings().hsvValue.set(190, 255); + pipeline.getSettings().outputShowThresholded = true; + pipeline.getSettings().outputShowMultipleTargets = true; + pipeline.getSettings().contourGroupingMode = ContourGroupingMode.Dual; + pipeline.getSettings().contourIntersection = ContourIntersectionDirection.Up; + + var frameProvider = + new FileFrameProvider( + TestUtils.getWPIImagePath(TestUtils.WPI2019Image.kCargoSideStraightDark72in), + TestUtils.WPI2019Image.FOV); + + frameProvider.setImageReloading(true); + + benchmarkPipeline(frameProvider, pipeline, 5); + } + + @Test + public void Reflective480pBenchmark() { + var pipeline = new ReflectivePipeline(); + pipeline.getSettings().hsvHue.set(60, 100); + pipeline.getSettings().hsvSaturation.set(200, 255); + pipeline.getSettings().hsvValue.set(200, 255); + pipeline.getSettings().outputShowThresholded = true; + + var frameProvider = + new FileFrameProvider( + TestUtils.getWPIImagePath(TestUtils.WPI2019Image.kCargoSideStraightDark72in), + TestUtils.WPI2019Image.FOV); + + frameProvider.setImageReloading(true); + + benchmarkPipeline(frameProvider, pipeline, 5); + } + + @Test + public void Reflective1920x1440Benchmark() { + var pipeline = new ReflectivePipeline(); + pipeline.getSettings().hsvHue.set(60, 100); + pipeline.getSettings().hsvSaturation.set(100, 255); + pipeline.getSettings().hsvValue.set(190, 255); + pipeline.getSettings().outputShowThresholded = true; + pipeline.getSettings().outputShowMultipleTargets = true; + pipeline.getSettings().contourGroupingMode = ContourGroupingMode.Dual; + pipeline.getSettings().contourIntersection = ContourIntersectionDirection.Up; + + var frameProvider = + new FileFrameProvider( + TestUtils.getWPIImagePath(TestUtils.WPI2019Image.kCargoStraightDark72in_HighRes), + TestUtils.WPI2019Image.FOV); + + frameProvider.setImageReloading(true); + + benchmarkPipeline(frameProvider, pipeline, 5); + } + + private static

void benchmarkPipeline( + FrameProvider frameProvider, P pipeline, int secondsToRun) { + CVMat.enablePrint(false); + // warmup for 5 loops. + System.out.println("Warming up for 5 loops..."); + for (int i = 0; i < 5; i++) { + pipeline.run(frameProvider.get()); + } + + final List processingTimes = new ArrayList<>(); + final List latencyTimes = new ArrayList<>(); + + var frameProps = frameProvider.get().frameStaticProperties; + + // begin benchmark + System.out.println("Beginning " + secondsToRun + " second benchmark at resolution " + frameProps.imageWidth + "x" + frameProps.imageHeight); + var benchmarkStartMillis = System.currentTimeMillis(); + do { + CVPipelineResult pipelineResult = pipeline.run(frameProvider.get()); + pipelineResult.release(); + processingTimes.add(MathUtils.roundTo(pipelineResult.processingMillis, 3)); + latencyTimes.add(MathUtils.roundTo(pipelineResult.getLatencyMillis(), 3)); + } while (System.currentTimeMillis() - benchmarkStartMillis < secondsToRun * 1000); + System.out.println("Benchmark complete."); + + var processingMin = Collections.min(processingTimes); + var processingMean = NumberListUtils.mean(processingTimes); + var processingMax = Collections.max(processingTimes); + + var latencyMin = Collections.min(latencyTimes); + var latencyMean = NumberListUtils.mean(latencyTimes); + var latencyMax = Collections.max(latencyTimes); + + String processingResult = + "Processing times - " + + "Min: " + + processingMin + + "ms (" + + 1000 / processingMin + + " FPS), " + + "Mean: " + + processingMean + + "ms (" + + 1000 / processingMean + + " FPS), " + + "Max: " + + processingMax + + "ms (" + + 1000 / processingMax + + " FPS)"; + System.out.println(processingResult); + String latencyResult = + "Latency times - " + + "Min: " + + latencyMin + + "ms (" + + 1000 / latencyMin + + " FPS), " + + "Mean: " + + latencyMean + + "ms (" + + 1000 / latencyMean + + " FPS), " + + "Max: " + + latencyMax + + "ms (" + + 1000 / latencyMax + + " FPS)"; + System.out.println(latencyResult); + } +}