diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java index fc057b790..ac6f93781 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java @@ -7,7 +7,7 @@ import org.opencv.core.Mat; * @param Pipeline result type */ public abstract class CVPipeline { - private CVPipelineSettings settings; + protected CVPipelineSettings settings; private Mat inputMat; protected Mat outputMat; diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java new file mode 100644 index 000000000..ffe35dae6 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java @@ -0,0 +1,219 @@ +package com.chameleonvision.classabstraction.pipeline; + +import com.chameleonvision.util.MathHandler; +import com.chameleonvision.vision.ImageFlipMode; +import com.chameleonvision.vision.camera.CameraValues; +import org.jetbrains.annotations.NotNull; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; + +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("WeakerAccess") +public class CVPipeline2d extends CVPipeline { + + private List foundContours_ = new ArrayList<>(); + private List filteredContours_ = new ArrayList<>(); + private List deSpeckledContours_ = new ArrayList<>(); + private List groupedContours_ = new ArrayList<>(); + + public CVPipeline2d(CVPipelineSettings settings) { + super(settings); + } + + @Override + void initPipeline() { + + } + + @Override + CVPipeline2d.CVPipeline2dResult runPipeline(Mat inputMat) { + var shouldFlip = settings.flipMode.equals(ImageFlipMode.BOTH); + var result = new CVPipeline2dResult(); + + // flip the image + if (shouldFlip) { + Core.flip(inputMat, inputMat, -1); + } + + foundContours_.clear(); + filteredContours_.clear(); + deSpeckledContours_.clear(); + groupedContours_.clear(); + + // HSV threshold the image + Scalar hsvLower = new Scalar(settings.hue.get(0).intValue(), settings.saturation.get(0).intValue(), settings.value.get(0).intValue()); + Scalar hsvUpper = new Scalar(settings.hue.get(1).intValue(), settings.saturation.get(1).intValue(), settings.value.get(1).intValue()); + hsvThreshold(inputImage, hsvThreshMat, settings.erode, settings.dilate); + + // Make sure we're BFR + if (settings.isBinary) { + Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); + } else { + inputImage.copyTo(outputImage); + } + + // search for contours + foundContours_ = findContours(hsvThreshMat); + if (foundContours_.size() < 1) { + return result; + } + + // filter contours by area, ratio and extent + filteredContours_ = filterContours(foundContours_, settings.area, settings.ratio, settings.extent); + if (filteredContours_.size() < 1) { + return result; + } + + // reject "speckle" contours + deSpeckledContours_ = rejectSpeckles(filteredContours_, settings.speckle.doubleValue()); + if (deSpeckledContours_.size() < 1) { + return result; + } + + // group targets + groupedContours_ = groupTargets(deSpeckledContours_, settings.targetIntersection, settings.targetGroup); + if (groupedContours_.size() < 1) { + return result; + } + + // sort targets down to our final target + var finalRect = sortTargetsToOne(groupedContours_, settings.sortMode); + result.RawPoint = finalRect; + result.IsValid = true; + switch (settings.calibrationMode) { + case None: + ///use the center of the USBCamera to find the pitch and yaw difference + result.CalibratedX = cameraValues.CenterX; + result.CalibratedY = cameraValues.CenterY; + break; + case Single: + // use the static point as a calibration method instead of the center + result.CalibratedX = settings.point.get(0).doubleValue(); + result.CalibratedY = settings.point.get(1).doubleValue(); + break; + case Dual: + // use the calculated line to find the difference in length between the point and the line + result.CalibratedX = (finalRect.center.y - settings.b) / settings.m; + result.CalibratedY = (finalRect.center.x * settings.m) + settings.b; + break; + } + + result.Pitch = cameraValues.CalculatePitch(finalRect.center.y, result.CalibratedY); + result.Yaw = cameraValues.CalculateYaw(finalRect.center.x, result.CalibratedX); + result.Area = finalRect.size.area(); + drawContour(outputImage, finalRect); + + return result; + } + + /** + * HSV Threshold a given image. Copies the HSV Thresholded image to the [dst] matrix with the given + * hsv settings and blur settings. Can also erode and dilate the image + * @param srcImage the source image, which is not mutated + * @param dst the destination image, which is mutated to save the result + * @param hsvLower the lower bound for the HSV settings + * @param hsvUpper the upper bound for the HSV settings + * @param kernel the kernal used to erode/dilate the image + * @param blur the size of the blur image + * @param shouldErode if we should erode + * @param shouldDilate if we should dilate + */ + public static void hsvThreshold(Mat srcImage, Mat dst, @NotNull Scalar hsvLower, + @NotNull Scalar hsvUpper, Mat kernel, Size blur, + boolean shouldErode, boolean shouldDilate) { + Imgproc.cvtColor(srcImage, dst, Imgproc.COLOR_RGB2HSV, 3); + Imgproc.blur(dst, dst, blur); + Core.inRange(dst, hsvLower, hsvUpper, dst); + if (shouldErode) { + Imgproc.erode(dst, dst, kernel); + } + if (shouldDilate) { + Imgproc.dilate(dst, dst, kernel); + } + dst.release(); + } + + /** + * Find contours from an image + * @param src the image we're looking at + * @param binaryMat a temporary image + * @param hierarchy the hierarchy of the image (just a new Mat();) + * @param emptyList a list to fill with stuff. Will be cleared + * @return the empty list, now full of contours + */ + public static List findContours(Mat src, Mat binaryMat, Mat hierarchy, List emptyList) { + src.copyTo(binaryMat); + emptyList.clear(); + Imgproc.findContours(binaryMat, emptyList, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); + binaryMat.release(); + return emptyList; + } + + public static List filterContours(List inputContours, List area, List ratio, List extent, CameraValues cameraValues) { + for (MatOfPoint Contour : inputContours) { + try { + double contourArea = Imgproc.contourArea(Contour); + double AreaRatio = (contourArea / cameraValues.ImageArea) * 100; + double minArea = (MathHandler.sigmoid(area.get(0))); + double maxArea = (MathHandler.sigmoid(area.get(1))); + if (AreaRatio < minArea || AreaRatio > maxArea) { + continue; + } + var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); + + var targetFullness = contourArea; + double minExtent = (double) (extent.get(0).doubleValue() * rect.size.area()) / 100; + double maxExtent = (double) (extent.get(1).doubleValue() * rect.size.area()) / 100; + if (targetFullness <= minExtent || contourArea >= maxExtent) { + continue; + } + Rect bb = Imgproc.boundingRect(Contour); + double aspectRatio = (bb.width / bb.height); + if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { + continue; + } + filteredContours.add(Contour); + } catch (Exception e) { + System.err.println("Error while filtering contours"); + e.printStackTrace(); + } + } + return filteredContours; + } + + @Override + Mat getOutputMat() { + return null; + } + + public static class CVPipeline2dSettings extends CVPipelineSettings { + double dualTargetCalibrationM = 1; + double dualTargetCalibrationB = 0; + } + + public static class CVPipeline2dResult { + public boolean hasTarget = false; + public ArrayList targets = new ArrayList<>(); // targets sorted by likelihood + + public CVPipeline2dResult(ArrayList targets, boolean hasTarget) { + this.targets = targets; + this.hasTarget = hasTarget; + } + + public CVPipeline2dResult() { + } + } + + public static class Target { + public boolean isValid = false; + public double calibratedX = 0.0; + public double calibratedY = 0.0; + public double pitch = 0.0; + public double yaw = 0.0; + public double area = 0.0; + RotatedRect rawPoint; + } + +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java deleted file mode 100644 index 20dd868ba..000000000 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.chameleonvision.classabstraction.pipeline; - -public class CVPipeline2dSettings extends CVPipelineSettings { -} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java new file mode 100644 index 000000000..7fce5f885 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java @@ -0,0 +1,33 @@ +package com.chameleonvision.classabstraction.pipeline; + +import org.opencv.core.Mat; + +public class CVPipeline3d extends CVPipeline { + + public CVPipeline3d(CVPipelineSettings settings) { + super(settings); + } + + @Override + void initPipeline() { + + } + + @Override + CVPipeline3d.CVPipeline3dResult runPipeline(Mat inputMat) { + return null; + } + + @Override + Mat getOutputMat() { + return null; + } + + public static class CVPipeline3dSettings extends CVPipelineSettings { + } + + public static class CVPipeline3dResult { + + } + +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java deleted file mode 100644 index b18a2e048..000000000 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.chameleonvision.classabstraction.pipeline; - -public abstract class CVPipeline3dSettings extends CVPipelineSettings { -} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java index c5f3281ae..79909b0fa 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java @@ -1,14 +1,13 @@ package com.chameleonvision.classabstraction.pipeline; -import com.chameleonvision.vision.CalibrationMode; -import com.chameleonvision.vision.SortMode; -import com.chameleonvision.vision.TargetGroup; -import com.chameleonvision.vision.TargetIntersection; +import com.chameleonvision.vision.*; import java.util.Arrays; import java.util.List; +@SuppressWarnings("ALL") public abstract class CVPipelineSettings { + ImageFlipMode flipMode = ImageFlipMode.NONE; List hue = Arrays.asList(50, 180); List saturation = Arrays.asList(50, 255); List value = Arrays.asList(50, 255); @@ -22,9 +21,10 @@ public abstract class CVPipelineSettings { SortMode sortMode = SortMode.Largest; TargetGroup targetGroup = TargetGroup.Single; TargetIntersection targetIntersection = TargetIntersection.Up; - double m = 1; - double b = 0; List point = Arrays.asList(0,0); CalibrationMode calibrationMode = CalibrationMode.None; + String nickname = ""; + double exposure = 50.0; + double brightness = 50.0; } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java index 200c38eb9..a0d985f40 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java @@ -9,7 +9,7 @@ public class DriverVisionPipeline extends CVPipeline { @Override void initPipeline() { - // set exposure/brightness of camera? + // TODO set exposure/brightness of camera } @Override diff --git a/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java b/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java new file mode 100644 index 000000000..5a778bd17 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java @@ -0,0 +1,5 @@ +package com.chameleonvision.vision; + +public enum ImageFlipMode { + NONE, VERTICAL, HORIZONTAL, BOTH +}