diff --git a/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue b/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue index d361105ad..25de890ac 100644 --- a/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue +++ b/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue @@ -11,6 +11,16 @@ const currentPipelineSettings = computed( () => useCameraSettingsStore().currentPipelineSettings ); +// TODO fix pv-range-slider so that store access doesn't need to be deferred +const contourArea = computed<[number, number]>({ + get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourArea) as [number, number], + set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourArea = v) +}); +const contourRatio = computed<[number, number]>({ + get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourRatio) as [number, number], + set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourRatio = v) +}); + const interactiveCols = computed(() => (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) && (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode) @@ -32,5 +42,42 @@ const interactiveCols = computed(() => :step="0.01" @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)" /> + + + + diff --git a/photon-core/src/main/java/org/photonvision/vision/opencv/Contour.java b/photon-core/src/main/java/org/photonvision/vision/opencv/Contour.java index 181de0b3b..1cf3e0d36 100644 --- a/photon-core/src/main/java/org/photonvision/vision/opencv/Contour.java +++ b/photon-core/src/main/java/org/photonvision/vision/opencv/Contour.java @@ -47,6 +47,16 @@ public class Contour implements Releasable { this.mat = mat; } + public Contour(Rect2d box) { + // no easy way to convert a Rect2d to Mat, diy it. Order is tl tr br bl + this.mat = + new MatOfPoint( + box.tl(), + new Point(box.x + box.width, box.y), + box.br(), + new Point(box.x, box.y + box.height)); + } + public MatOfPoint2f getMat2f() { if (mat2f == null) { mat2f = new MatOfPoint2f(mat.toArray()); diff --git a/photon-core/src/main/java/org/photonvision/vision/opencv/ContourSortMode.java b/photon-core/src/main/java/org/photonvision/vision/opencv/ContourSortMode.java index 2ceff908e..531e3b332 100644 --- a/photon-core/src/main/java/org/photonvision/vision/opencv/ContourSortMode.java +++ b/photon-core/src/main/java/org/photonvision/vision/opencv/ContourSortMode.java @@ -25,15 +25,15 @@ public enum ContourSortMode { Comparator.comparingDouble(PotentialTarget::getArea) .reversed()), // reversed so that zero index has the largest size Smallest(Largest.getComparator().reversed()), - Highest(Comparator.comparingDouble(rect -> rect.getMinAreaRect().center.y)), + Highest(Comparator.comparingDouble(tgt -> tgt.getMinAreaRect().center.y)), Lowest(Highest.getComparator().reversed()), - Leftmost(Comparator.comparingDouble(target -> target.getMinAreaRect().center.x * -1)), + Leftmost(Comparator.comparingDouble(tgt -> tgt.getMinAreaRect().center.x * -1)), Rightmost(Leftmost.getComparator().reversed()), Centermost( Comparator.comparingDouble( - rect -> - (Math.pow(rect.getMinAreaRect().center.y, 2) - + Math.pow(rect.getMinAreaRect().center.x, 2)))); + tgt -> + (Math.pow(tgt.getMinAreaRect().center.y, 2) + + Math.pow(tgt.getMinAreaRect().center.x, 2)))); private final Comparator m_comparator; diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterObjectDetectionsPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterObjectDetectionsPipe.java new file mode 100644 index 000000000..c2fff471e --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterObjectDetectionsPipe.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.pipe.impl; + +import java.util.ArrayList; +import java.util.List; +import org.photonvision.common.util.numbers.DoubleCouple; +import org.photonvision.vision.frame.FrameStaticProperties; +import org.photonvision.vision.pipe.CVPipe; + +public class FilterObjectDetectionsPipe + extends CVPipe< + List, + List, + FilterObjectDetectionsPipe.FilterContoursParams> { + List m_filteredContours = new ArrayList<>(); + + @Override + protected List process(List in) { + m_filteredContours.clear(); + for (var contour : in) { + filterContour(contour); + } + + return m_filteredContours; + } + + private void filterContour(NeuralNetworkPipeResult contour) { + var boc = contour.box; + + // Area filtering + double areaPercentage = boc.area() / params.getFrameStaticProperties().imageArea * 100.0; + double minAreaPercentage = params.getArea().getFirst(); + double maxAreaPercentage = params.getArea().getSecond(); + if (areaPercentage < minAreaPercentage || areaPercentage > maxAreaPercentage) return; + + // Aspect ratio filtering; much simpler since always axis-aligned + double aspectRatio = boc.width / boc.height; + if (aspectRatio < params.getRatio().getFirst() || aspectRatio > params.getRatio().getSecond()) + return; + + m_filteredContours.add(contour); + } + + public static class FilterContoursParams { + private final DoubleCouple m_area; + private final DoubleCouple m_ratio; + private final FrameStaticProperties m_frameStaticProperties; + public final boolean isLandscape; + + public FilterContoursParams( + DoubleCouple area, + DoubleCouple ratio, + FrameStaticProperties camProperties, + boolean isLandscape) { + this.m_area = area; + this.m_ratio = ratio; + this.m_frameStaticProperties = camProperties; + this.isLandscape = isLandscape; + } + + public DoubleCouple getArea() { + return m_area; + } + + public DoubleCouple getRatio() { + return m_ratio; + } + + public FrameStaticProperties getFrameStaticProperties() { + return m_frameStaticProperties; + } + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/SortContoursPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/SortContoursPipe.java index 7bdc9bf0d..b084c75c4 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/SortContoursPipe.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/SortContoursPipe.java @@ -42,6 +42,7 @@ public class SortContoursPipe if (params.getSortMode() != ContourSortMode.Centermost) { m_sortedContours.sort(params.getSortMode().getComparator()); } else { + // we need knowledge of camera properties to calculate this distance -- do it ourselves m_sortedContours.sort(Comparator.comparingDouble(this::calcSquareCenterDistance)); } } @@ -50,10 +51,10 @@ public class SortContoursPipe m_sortedContours.subList(0, Math.min(in.size(), params.getMaxTargets()))); } - private double calcSquareCenterDistance(PotentialTarget rect) { + private double calcSquareCenterDistance(PotentialTarget tgt) { return Math.sqrt( - Math.pow(params.getCamProperties().centerX - rect.getMinAreaRect().center.x, 2) - + Math.pow(params.getCamProperties().centerY - rect.getMinAreaRect().center.y, 2)); + Math.pow(params.getCamProperties().centerX - tgt.getMinAreaRect().center.x, 2) + + Math.pow(params.getCamProperties().centerY - tgt.getMinAreaRect().center.y, 2)); } public static class SortContoursParams { diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java index e211b42de..928e905fb 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java @@ -26,6 +26,8 @@ import org.photonvision.vision.pipeline.result.CVPipelineResult; public abstract class CVPipeline implements Releasable { + static final int MAX_MULTI_TARGET_RESULTS = 10; + protected S settings; protected FrameStaticProperties frameStaticProperties; protected QuirkyCamera cameraQuirks; diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java index 190de9e68..8e9292491 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java @@ -109,7 +109,7 @@ public class ColoredShapePipeline SortContoursPipe.SortContoursParams sortContoursParams = new SortContoursPipe.SortContoursParams( settings.contourSortMode, - settings.outputShowMultipleTargets ? 5 : 1, + settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1, frameStaticProperties); // TODO don't hardcode? sortContoursPipe.setParams(sortContoursParams); diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java index 58154bd26..10cf87d1b 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java @@ -17,21 +17,26 @@ package org.photonvision.vision.pipeline; -import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.FrameThresholdType; +import org.photonvision.vision.opencv.DualOffsetValues; import org.photonvision.vision.pipe.CVPipe.CVPipeResult; import org.photonvision.vision.pipe.impl.*; import org.photonvision.vision.pipe.impl.RknnDetectionPipe.RknnDetectionPipeParams; import org.photonvision.vision.pipeline.result.CVPipelineResult; +import org.photonvision.vision.target.PotentialTarget; +import org.photonvision.vision.target.TargetOrientation; import org.photonvision.vision.target.TrackedTarget; -import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters; public class ObjectDetectionPipeline extends CVPipeline { private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe(); private final RknnDetectionPipe rknnPipe = new RknnDetectionPipe(); + private final SortContoursPipe sortContoursPipe = new SortContoursPipe(); + private final Collect2dTargetsPipe collect2dTargetsPipe = new Collect2dTargetsPipe(); + private final FilterObjectDetectionsPipe filterContoursPipe = new FilterObjectDetectionsPipe(); private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE; @@ -52,6 +57,38 @@ public class ObjectDetectionPipeline params.confidence = settings.confidence; params.nms = settings.nms; rknnPipe.setParams(params); + + DualOffsetValues dualOffsetValues = + new DualOffsetValues( + settings.offsetDualPointA, + settings.offsetDualPointAArea, + settings.offsetDualPointB, + settings.offsetDualPointBArea); + + SortContoursPipe.SortContoursParams sortContoursParams = + new SortContoursPipe.SortContoursParams( + settings.contourSortMode, + settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1, + frameStaticProperties); + sortContoursPipe.setParams(sortContoursParams); + + var filterContoursParams = + new FilterObjectDetectionsPipe.FilterContoursParams( + settings.contourArea, + settings.contourRatio, + frameStaticProperties, + settings.contourTargetOrientation == TargetOrientation.Landscape); + filterContoursPipe.setParams(filterContoursParams); + + Collect2dTargetsPipe.Collect2dTargetsParams collect2dTargetsParams = + new Collect2dTargetsPipe.Collect2dTargetsParams( + settings.offsetRobotOffsetMode, + settings.offsetSinglePoint, + dualOffsetValues, + settings.contourTargetOffsetPointEdge, + settings.contourTargetOrientation, + frameStaticProperties); + collect2dTargetsPipe.setParams(collect2dTargetsParams); } @Override @@ -60,31 +97,35 @@ public class ObjectDetectionPipeline // ***************** change based on backend *********************** - CVPipeResult> ret = rknnPipe.run(input_frame.colorImage); - sumPipeNanosElapsed += ret.nanosElapsed; + CVPipeResult> rknnResult = rknnPipe.run(input_frame.colorImage); + sumPipeNanosElapsed += rknnResult.nanosElapsed; List targetList; - targetList = ret.output; var names = rknnPipe.getClassNames(); input_frame.colorImage.getMat().copyTo(input_frame.processedImage.getMat()); // ***************** change based on backend *********************** - List targets = new ArrayList<>(); + var filterContoursResult = filterContoursPipe.run(rknnResult.output); + sumPipeNanosElapsed += filterContoursResult.nanosElapsed; - for (var t : targetList) { - targets.add( - new TrackedTarget( - t, - new TargetCalculationParameters( - false, null, null, null, null, frameStaticProperties))); - } + CVPipeResult> sortContoursResult = + sortContoursPipe.run( + filterContoursResult.output.stream() + .map(shape -> new PotentialTarget(shape)) + .collect(Collectors.toList())); + sumPipeNanosElapsed += sortContoursResult.nanosElapsed; + + CVPipeResult> collect2dTargetsResult = + collect2dTargetsPipe.run(sortContoursResult.output); + sumPipeNanosElapsed += collect2dTargetsResult.nanosElapsed; var fpsResult = calculateFPSPipe.run(null); var fps = fpsResult.output; - return new CVPipelineResult(sumPipeNanosElapsed, fps, targets, input_frame, names); + return new CVPipelineResult( + sumPipeNanosElapsed, fps, collect2dTargetsResult.output, input_frame, names); } @Override diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java index e961488b7..e03889785 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java @@ -64,29 +64,6 @@ public class ReflectivePipeline extends CVPipeline m_subContours; public final CVShape shape; + // additional metadata about object detections we need to keep around + public final double confidence; + public final int clsId; + public PotentialTarget(Contour inputContour) { this(inputContour, List.of()); } @@ -41,12 +47,26 @@ public class PotentialTarget implements Releasable { m_mainContour = inputContour; m_subContours = new ArrayList<>(subContours); this.shape = shape; + this.clsId = -1; + this.confidence = -1; } public PotentialTarget(Contour inputContour, CVShape shape) { this(inputContour, List.of(), shape); } + public PotentialTarget(NeuralNetworkPipeResult det) { + this.shape = new CVShape(new Contour(det.box), ContourShape.Quadrilateral); + this.m_mainContour = this.shape.getContour(); + m_subContours = List.of(); + this.clsId = det.classIdx; + this.confidence = det.confidence; + } + + public PotentialTarget(CVShape cvShape) { + this(cvShape.getContour(), cvShape); + } + public RotatedRect getMinAreaRect() { return m_mainContour.getMinAreaRect(); } @@ -61,7 +81,7 @@ public class PotentialTarget implements Releasable { for (var sc : m_subContours) { sc.release(); } - m_subContours.clear(); + if (!m_subContours.isEmpty()) m_subContours.clear(); if (shape != null) shape.release(); } } diff --git a/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java b/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java index 396e92a22..bf7e2e6c3 100644 --- a/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java +++ b/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java @@ -27,7 +27,6 @@ import org.opencv.core.Mat; import org.opencv.core.MatOfPoint; import org.opencv.core.MatOfPoint2f; import org.opencv.core.Point; -import org.opencv.core.Rect2d; import org.opencv.core.RotatedRect; import org.photonvision.common.util.SerializationUtils; import org.photonvision.common.util.math.MathUtils; @@ -39,7 +38,6 @@ import org.photonvision.vision.opencv.CVShape; import org.photonvision.vision.opencv.Contour; import org.photonvision.vision.opencv.DualOffsetValues; import org.photonvision.vision.opencv.Releasable; -import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult; public class TrackedTarget implements Releasable { public final Contour m_mainContour; @@ -76,6 +74,9 @@ public class TrackedTarget implements Releasable { this.m_subContours = origTarget.m_subContours; this.m_shape = shape; calculateValues(params); + + this.m_classId = origTarget.clsId; + this.m_confidence = origTarget.confidence; } public TrackedTarget( @@ -159,47 +160,6 @@ public class TrackedTarget implements Releasable { m_robotOffsetPoint = new Point(); } - public TrackedTarget( - Rect2d box, int class_id, double confidence, TargetCalculationParameters params) { - m_targetOffsetPoint = new Point(box.x + box.width / 2.0, box.y + box.height / 2.0); - m_robotOffsetPoint = new Point(); - - var yawPitch = - TargetCalculations.calculateYawPitch( - params.cameraCenterPoint.x, - box.x + box.width / 2.0, - params.horizontalFocalLength, - params.cameraCenterPoint.y, - box.y + box.height / 2.0, - params.verticalFocalLength); - m_yaw = yawPitch.getFirst(); - m_pitch = yawPitch.getSecond(); - - Point[] cornerPoints = - new Point[] { - // Box.x/y is the top-left corner, not the center - new Point(box.x, box.y), // tl - new Point(box.x + box.width, box.y), // tr - new Point(box.x + box.width, box.y + box.height), // br - new Point(box.x, box.y + box.height), // bl - }; - - m_targetCorners = List.of(cornerPoints); - MatOfPoint contourMat = new MatOfPoint(cornerPoints); - m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints); - - m_mainContour = new Contour(contourMat); - m_area = m_mainContour.getArea() / params.imageArea * 100; - - m_classId = class_id; - m_confidence = confidence; - } - - public TrackedTarget( - NeuralNetworkPipeResult t, TargetCalculationParameters targetCalculationParameters) { - this(t.box, t.classIdx, t.confidence, targetCalculationParameters); - } - /** * @return Returns the confidence of the detection ranging from 0 - 1. */