From 80f711c37c5cf169addb5375894e6afd519dbdfe Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 10 Nov 2019 11:47:56 -0500 Subject: [PATCH] Begin work on VisionProcess, refinement of CVPipeline interactions --- .../classabstraction/VisionManager.java | 61 +++++++++++++++++ .../classabstraction/VisionProcess.java | 41 +++++++++++ .../classabstraction/camera/CameraConfig.java | 10 +++ .../camera/USBCameraProcess.java | 8 +-- .../classabstraction/pipeline/CVPipeline.java | 22 ++++-- .../pipeline/CVPipeline2d.java | 29 +++----- .../pipeline/CVPipeline2dSettings.java | 29 ++++++++ .../pipeline/CVPipeline3d.java | 66 +++++++++--------- .../pipeline/CVPipeline3dSettings.java | 5 ++ .../pipeline/CVPipelineResult.java | 12 +++- .../pipeline/CVPipelineSettings.java | 29 ++------ .../pipeline/DriverVisionPipeline.java | 30 +++++--- .../vision/process/VisionProcess.java | 68 ++++++++++++++++--- 13 files changed, 300 insertions(+), 110 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java b/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java new file mode 100644 index 000000000..e5d735f2f --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java @@ -0,0 +1,61 @@ +package com.chameleonvision.classabstraction; + +import com.chameleonvision.classabstraction.camera.USBCameraProcess; +import com.chameleonvision.settings.SettingsManager; +import com.chameleonvision.util.FileHelper; +import com.chameleonvision.vision.camera.CameraDeserializer; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import edu.wpi.cscore.UsbCamera; +import edu.wpi.cscore.UsbCameraInfo; +import org.opencv.videoio.VideoCapture; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; + +public class VisionManager { + + private static final Path CamConfigPath = Paths.get(SettingsManager.SettingsPath.toString(), "cameras"); + + public static final LinkedHashMap UsbCameraInfosByCameraName = new LinkedHashMap<>(); + public static final LinkedHashMap VisionProcessesByCameraName = new LinkedHashMap<>(); + + + public static boolean initializeSources() { + int suffix = 0; + for (UsbCameraInfo info : UsbCamera.enumerateUsbCameras()) { + VideoCapture cap = new VideoCapture(info.dev); + if (cap.isOpened()) { + cap.release(); + String name = info.name; + while (UsbCameraInfosByCameraName.containsKey(name)) { + suffix++; + name = String.format("%s (%d)", name, suffix); + } + UsbCameraInfosByCameraName.put(name, info); + } + } + + if (UsbCameraInfosByCameraName.isEmpty()) { + return false; + } + + FileHelper.CheckPath(CamConfigPath); + UsbCameraInfosByCameraName.forEach((cameraName, cameraInfo) -> { + Path cameraConfigPath = Paths.get(CamConfigPath.toString(), String.format("%s.json", cameraName)); + File cameraConfigFile = new File(cameraConfigPath.toString()); + if (cameraConfigFile.exists() && cameraConfigFile.length() != 0) { + try { + Gson gson = new GsonBuilder()().registerTypeAdapter(USBCameraProcess.class, new CameraDeserializer()) + } + } + }) + + } + + public static void initializeProcesses() { + + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java new file mode 100644 index 000000000..eb54fcf7c --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java @@ -0,0 +1,41 @@ +package com.chameleonvision.classabstraction; + +import com.chameleonvision.classabstraction.camera.CameraProcess; +import com.chameleonvision.classabstraction.pipeline.CVPipeline; +import com.chameleonvision.classabstraction.pipeline.CVPipelineSettings; +import com.chameleonvision.classabstraction.pipeline.DriverVisionPipeline; + +import java.util.ArrayList; +import java.util.List; + +public class VisionProcess { + + private final CameraProcess cameraProcess; + private final List pipelines = new ArrayList<>(); + private CVPipeline currentPipeline; + + private final CVPipelineSettings driverVisionSettings = new CVPipelineSettings(); + + public VisionProcess(CameraProcess cameraProcess) { + this.cameraProcess = cameraProcess; + + pipelines.add(new DriverVisionPipeline(() -> driverVisionSettings)); + setPipeline(pipelines.get(0)); + } + + public void setPipeline(int pipelineIndex) { + CVPipeline newPipeline = pipelines.get(pipelineIndex); + if (newPipeline != null) { + setPipeline(newPipeline); + } + } + + public void setPipeline(CVPipeline pipeline) { + currentPipeline = pipeline; + currentPipeline.initPipeline(cameraProcess); + } + + public CVPipeline getCurrentPipeline() { + return currentPipeline; + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java new file mode 100644 index 000000000..830b9a358 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java @@ -0,0 +1,10 @@ +package com.chameleonvision.classabstraction.camera; + +public class CameraConfig { + public double FOV; + public String Path; + public String Name; + public String Nickname; + public com.chameleonvision.vision.camera.StreamDivisor StreamDivisor; +// public +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java index 37b259315..be5b506ef 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java @@ -14,11 +14,11 @@ public class USBCameraProcess implements CameraProcess { private Mat imageBuffer = new Mat(); public final CameraProperties properties; - public USBCameraProcess(UsbCamera camera, double fov) { + public USBCameraProcess(UsbCamera camera, CameraConfig config) { baseCamera = camera; cvSink = CameraServer.getInstance().getVideo(baseCamera); VideoMode vidMode = new VideoMode(VideoMode.PixelFormat.kYUYV, 640, 480, 60); - properties = new CameraProperties(baseCamera, fov); + properties = new CameraProperties(baseCamera, config.FOV); } @Override @@ -41,7 +41,7 @@ public class USBCameraProcess implements CameraProcess { try { baseCamera.setExposureManual(exposure); } catch (VideoException e) { - System.err.println("USBCameraProcess Does not support exposure change"); + System.err.println("Current camera does not support exposure change"); } } @@ -50,7 +50,7 @@ public class USBCameraProcess implements CameraProcess { try { baseCamera.setBrightness(brightness); } catch (VideoException e) { - System.err.println("USBCameraProcess Does not support brightness change"); + System.err.println("Current camera does not support brightness change"); } } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java index e7c686fc1..3706bf812 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java @@ -3,19 +3,29 @@ package com.chameleonvision.classabstraction.pipeline; import com.chameleonvision.classabstraction.camera.CameraProcess; import org.opencv.core.Mat; +import java.util.function.Supplier; + /** * * @param Pipeline result type */ public abstract class CVPipeline { - protected S settings; - private Mat inputMat; protected Mat outputMat; + CameraProcess cameraProcess; + final Supplier settingsSupplier; - public CVPipeline(S settings) { - this.settings = settings; + public CVPipeline(Supplier settingsSupplier) { + this.settingsSupplier = settingsSupplier; } - abstract void initPipeline(CameraProcess camera); - abstract R runPipeline(Mat inputMat); + public S getSettings() { + return settingsSupplier.get(); + } + + public void initPipeline(CameraProcess camera) { + cameraProcess = camera; + cameraProcess.setExposure((int) getSettings().exposure); + cameraProcess.setBrightness((int) getSettings().brightness); + } + abstract public R runPipeline(Mat inputMat); } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java index bba5566c0..8851a5422 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java @@ -1,6 +1,5 @@ package com.chameleonvision.classabstraction.pipeline; -import com.chameleonvision.classabstraction.camera.CameraProcess; import com.chameleonvision.classabstraction.camera.CameraStaticProperties; import com.chameleonvision.classabstraction.pipeline.pipes.*; import com.chameleonvision.vision.ImageRotation; @@ -9,29 +8,26 @@ import org.opencv.core.*; import org.opencv.imgproc.Imgproc; import java.util.List; +import java.util.function.Supplier; + +import static com.chameleonvision.classabstraction.pipeline.CVPipeline2d.*; @SuppressWarnings("WeakerAccess") -public class CVPipeline2d extends CVPipeline { - - private CameraProcess cameraProcess; +public class CVPipeline2d extends CVPipeline { private Mat rawCameraMat = new Mat(); private Mat hsvOutputMat = new Mat(); - public CVPipeline2d(CVPipeline2dSettings settings) { - super(settings); + public CVPipeline2d(Supplier settingsSupplier) { + super(settingsSupplier); } @Override - void initPipeline(CameraProcess cam) { - cameraProcess = cam; - } - - @Override - CVPipeline2d.CVPipeline2dResult runPipeline(Mat inputMat) { + public CVPipeline2dResult runPipeline(Mat inputMat) { long totalProcessTimeNanos = 0; StringBuilder procTimeStringBuilder = new StringBuilder(); + var settings = settingsSupplier.get(); CameraStaticProperties camProps = cameraProcess.getProperties().staticProperties; inputMat.copyTo(rawCameraMat); @@ -123,16 +119,9 @@ public class CVPipeline2d extends CVPipeline { public CVPipeline2dResult(List targets, Mat outputMat) { - this.targets = targets; - this.hasTarget = !targets.isEmpty(); - this.outputMat = outputMat; + super(targets, outputMat); } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java new file mode 100644 index 000000000..43893215d --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java @@ -0,0 +1,29 @@ +package com.chameleonvision.classabstraction.pipeline; + +import com.chameleonvision.vision.CalibrationMode; +import com.chameleonvision.vision.SortMode; +import com.chameleonvision.vision.TargetGroup; +import com.chameleonvision.vision.TargetIntersection; + +import java.util.Arrays; +import java.util.List; + +public class CVPipeline2dSettings extends CVPipelineSettings { + public List hue = Arrays.asList(50, 180); + public List saturation = Arrays.asList(50, 255); + public List value = Arrays.asList(50, 255); + public boolean erode = false; + public boolean dilate = false; + public List area = Arrays.asList(0.0, 100.0); + public List ratio = Arrays.asList(0.0, 20.0); + public List extent = Arrays.asList(0, 100); + public Number speckle = 5; + public boolean isBinary = false; + public SortMode sortMode = SortMode.Largest; + public TargetGroup targetGroup = TargetGroup.Single; + public TargetIntersection targetIntersection = TargetIntersection.Up; + public List point = Arrays.asList(0, 0); + public CalibrationMode calibrationMode = CalibrationMode.None; + public double dualTargetCalibrationM = 1; + public double dualTargetCalibrationB = 0; +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java index 9fa8a6bd1..66029cfe0 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java @@ -1,33 +1,33 @@ -//package com.chameleonvision.classabstraction.pipeline; -// -//import org.opencv.core.Mat; -// -//public class CVPipeline3d extends CVPipeline { -// -// public CVPipeline3d(CVPipelineSettings settings) { -// super(settings); -// } -// -// @Override -// void initPipeline() { -// -// } -// -// @Override -// CVPipeline3d.CVPipeline3dResult runPipeline(Mat inputMat) { -// return null; -// } -// -// @Override -// Mat getOutputMat() { -// return null; -// } -// -// public static class CVPipeline3dSettings extends CVPipelineSettings { -// } -// -// public static class CVPipeline3dResult { -// -// } -// -//} +package com.chameleonvision.classabstraction.pipeline; + +import com.chameleonvision.classabstraction.camera.CameraProcess; +import org.opencv.core.Mat; + +import java.util.List; +import java.util.function.Supplier; + +import static com.chameleonvision.classabstraction.pipeline.CVPipeline3d.*; + +public class CVPipeline3d extends CVPipeline { + + public CVPipeline3d(Supplier settingsSupplier) { + super(settingsSupplier); + } + + @Override + public CVPipeline3dResult runPipeline(Mat inputMat) { + return null; + } + + + public static class CVPipeline3dResult extends CVPipelineResult { + public CVPipeline3dResult(List targets, Mat outputMat) { + super(targets, outputMat); + } + } + + public static class Target3d { + // TODO: Define 3d-specific target data + } + +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java new file mode 100644 index 000000000..e9a5be7ba --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java @@ -0,0 +1,5 @@ +package com.chameleonvision.classabstraction.pipeline; + +public class CVPipeline3dSettings extends CVPipeline2dSettings { + // TODO: +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java index eb639ea53..3ffe26d9d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java @@ -5,7 +5,13 @@ import org.opencv.core.Mat; import java.util.List; public abstract class CVPipelineResult { - List targets; - boolean hasTarget; - Mat outputMat; + public final List targets; + public final boolean hasTarget; + public final Mat outputMat = new Mat(); + + public CVPipelineResult(List targets, Mat outputMat) { + this.targets = targets; + hasTarget = targets != null && !targets.isEmpty(); + outputMat.copyTo(this.outputMat); + } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java index 79909b0fa..644461a9d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java @@ -2,29 +2,10 @@ package com.chameleonvision.classabstraction.pipeline; import com.chameleonvision.vision.*; -import java.util.Arrays; -import java.util.List; - @SuppressWarnings("ALL") -public abstract class CVPipelineSettings { - ImageFlipMode flipMode = ImageFlipMode.NONE; - List hue = Arrays.asList(50, 180); - List saturation = Arrays.asList(50, 255); - List value = Arrays.asList(50, 255); - boolean erode = false; - boolean dilate = false; - List area = Arrays.asList(0.0, 100.0); - List ratio = Arrays.asList(0.0, 20.0); - List extent = Arrays.asList(0, 100); - Number speckle = 5; - boolean isBinary = false; - SortMode sortMode = SortMode.Largest; - TargetGroup targetGroup = TargetGroup.Single; - TargetIntersection targetIntersection = TargetIntersection.Up; - List point = Arrays.asList(0,0); - CalibrationMode calibrationMode = CalibrationMode.None; - - String nickname = ""; - double exposure = 50.0; - double brightness = 50.0; +public class CVPipelineSettings { + public ImageFlipMode flipMode = ImageFlipMode.NONE; + public String nickname = ""; + public double exposure = 50.0; + public double brightness = 50.0; } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java index 82a8ee333..c59a5ea83 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java @@ -3,26 +3,34 @@ package com.chameleonvision.classabstraction.pipeline; import com.chameleonvision.classabstraction.camera.CameraProcess; import org.opencv.core.Mat; -public class DriverVisionPipeline extends CVPipeline { - public DriverVisionPipeline(CVPipelineSettings settings) { - super(settings); +import java.util.List; +import java.util.function.Supplier; + +import static com.chameleonvision.classabstraction.pipeline.DriverVisionPipeline.*; + +public class DriverVisionPipeline extends CVPipeline { + + public DriverVisionPipeline(Supplier settingsSupplier) { + super(settingsSupplier); } @Override - void initPipeline(CameraProcess camera) { - // TODO: set camera to driver mode + public void initPipeline(CameraProcess camera) { + camera.setBrightness((int) getSettings().brightness); + camera.setExposure((int) getSettings().exposure); } @Override - DriverPipelineResult runPipeline(Mat inputMat) { - return new DriverPipelineResult(inputMat); + public DriverPipelineResult runPipeline(Mat inputMat) { + + inputMat.copyTo(outputMat); + + return new DriverPipelineResult(null, inputMat); } public static class DriverPipelineResult extends CVPipelineResult { - public DriverPipelineResult(Mat outputMat) { - this.hasTarget = false; - this.targets = null; - outputMat.copyTo(this.outputMat); + public DriverPipelineResult(List targets, Mat outputMat) { + super(targets, outputMat); } } } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java index 44caa86f9..30d460ce3 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java @@ -1,5 +1,7 @@ package com.chameleonvision.vision.process; +import com.chameleonvision.classabstraction.pipeline.CVPipeline2d; +import com.chameleonvision.classabstraction.pipeline.DriverVisionPipeline; import com.chameleonvision.settings.SettingsManager; import com.chameleonvision.vision.Orientation; import com.chameleonvision.vision.Pipeline; @@ -12,6 +14,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import static com.chameleonvision.classabstraction.pipeline.CVPipeline2d.*; + public class VisionProcess implements Runnable { private final String cameraName; @@ -29,6 +33,10 @@ public class VisionProcess implements Runnable { // chameleon specific private Pipeline currentPipeline; private CVProcess cvProcess; + + private CVPipeline2d cvPipeline2d; + private DriverVisionPipeline driverVisionPipeline; + // pipeline process items // private List foundContours = new ArrayList<>(); // private List filteredContours = new ArrayList<>(); @@ -41,7 +49,6 @@ public class VisionProcess implements Runnable { public VisionProcess(CameraProcess cameraProcess) { - // USBCameraProcess settings cvProcess = new StandardCVProcess(cameraProcess.getCamVals()); this.cameraProcess = cameraProcess; // new USBCameraProcess(cameraProcess); @@ -95,15 +102,58 @@ public class VisionProcess implements Runnable { } } + private CVPipeline2dSettings pipelineTo2dSettings(Pipeline pipeline) { + CVPipeline2dSettings settings = new CVPipeline2dSettings(); + settings.hue = pipeline.hue; + settings.saturation = pipeline.saturation; + settings.value = pipeline.value; + settings.erode = pipeline.erode; + settings.dilate = pipeline.dilate; + settings.area = pipeline.area; + settings.ratio = pipeline.ratio; + settings.extent = pipeline.extent; + settings.speckle = pipeline.speckle; + settings.isBinary = pipeline.isBinary; + settings.sortMode = pipeline.sortMode; + settings.targetGroup = pipeline.targetGroup; + settings.targetIntersection = pipeline.targetIntersection; + settings.point = pipeline.point; + settings.calibrationMode = pipeline.calibrationMode; + settings.nickname = pipeline.nickname; + settings.exposure = pipeline.exposure; + settings.brightness = pipeline.brightness; + settings.dualTargetCalibrationM = pipeline.m; + settings.dualTargetCalibrationB = pipeline.b; + + return settings; + } + private PipelineResult runVisionProcess(Mat inputImage, Mat outputImage) { - return cvProcess.runPipeline( - currentPipeline, - inputImage, - outputImage, - cameraProcess.getCamVals(), - currentPipeline.orientation.equals(Orientation.Inverted), - cameraProcess.getDriverMode() - ); + + if (cvPipeline2d == null) { + cvPipeline2d = new CVPipeline2d(() -> pipelineTo2dSettings(currentPipeline)); + } + CVPipeline2dResult result = cvPipeline2d.runPipeline(inputImage); + result.outputMat.copyTo(outputImage); + + PipelineResult pipeResult = new PipelineResult(); + pipeResult.IsValid = result.hasTarget; + if (!pipeResult.IsValid) { + pipeResult.CalibratedX = 0; + pipeResult.CalibratedY = 0; + pipeResult.Pitch = 0; + pipeResult.Yaw = 0; + pipeResult.Area = 0; + } else { + Target t = result.targets.get(0); + pipeResult.CalibratedX = t.calibratedX; + pipeResult.CalibratedY = t.calibratedY; + pipeResult.Pitch = t.pitch; + pipeResult.Yaw = t.yaw; + pipeResult.Area = t.area; + } + + return pipeResult; } @Override