From 24313ba45b89307e0e46a0c8925980090f7630b2 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 2 Nov 2019 10:10:02 -0700 Subject: [PATCH 01/84] Rename Camera and CamProcess to USBCamera and USBCameraProcess Further changes: - Add CameraProcess interface - Change VisionProcess to pass the interface as parameter --- .../main/java/com/chameleonvision/Main.java | 2 +- .../network/NetworkManager.java | 4 +- .../settings/SettingsManager.java | 20 +-- .../vision/camera/CameraDeserializer.java | 13 +- .../vision/camera/CameraManager.java | 78 +++++----- .../vision/camera/CameraSerializer.java | 26 ++-- .../vision/camera/CameraValues.java | 4 +- .../camera/{Camera.java => USBCamera.java} | 30 ++-- .../vision/process/CameraProcess.java | 86 +++-------- .../vision/process/USBCameraProcess.java | 144 ++++++++++++++++++ .../vision/process/VisionProcess.java | 60 ++++---- .../chameleonvision/web/ServerHandler.java | 24 ++- 12 files changed, 290 insertions(+), 201 deletions(-) rename Main/src/main/java/com/chameleonvision/vision/camera/{Camera.java => USBCamera.java} (89%) create mode 100644 Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java diff --git a/Main/src/main/java/com/chameleonvision/Main.java b/Main/src/main/java/com/chameleonvision/Main.java index 6941cb7ea..7e2046a19 100644 --- a/Main/src/main/java/com/chameleonvision/Main.java +++ b/Main/src/main/java/com/chameleonvision/Main.java @@ -145,7 +145,7 @@ public class Main { if (ntClientModeServer != null) { NetworkTableInstance.getDefault().startClient(ntClientModeServer); } else { - NetworkTableInstance.getDefault().startClientTeam(SettingsManager.GeneralSettings.teamNumber); + NetworkTableInstance.getDefault().startClientTeam(SettingsManager.generalSettings.teamNumber); } } diff --git a/Main/src/main/java/com/chameleonvision/network/NetworkManager.java b/Main/src/main/java/com/chameleonvision/network/NetworkManager.java index ad8e1cc15..a60db1c15 100644 --- a/Main/src/main/java/com/chameleonvision/network/NetworkManager.java +++ b/Main/src/main/java/com/chameleonvision/network/NetworkManager.java @@ -43,7 +43,7 @@ public class NetworkManager { e.printStackTrace(); } - var teamBytes = NetworkManager.GetTeamNumberIPBytes(SettingsManager.GeneralSettings.teamNumber); + var teamBytes = NetworkManager.GetTeamNumberIPBytes(SettingsManager.generalSettings.teamNumber); if (interfaces.size() > 0) { for (var inetface : interfaces) { @@ -85,7 +85,7 @@ public class NetworkManager { return true; } - var genSettings = SettingsManager.GeneralSettings; + var genSettings = SettingsManager.generalSettings; boolean isStatic = genSettings.connectionType.equals(NetworkIPMode.STATIC); if (isStatic) { diff --git a/Main/src/main/java/com/chameleonvision/settings/SettingsManager.java b/Main/src/main/java/com/chameleonvision/settings/SettingsManager.java index d26444630..52fde8312 100644 --- a/Main/src/main/java/com/chameleonvision/settings/SettingsManager.java +++ b/Main/src/main/java/com/chameleonvision/settings/SettingsManager.java @@ -14,36 +14,36 @@ import java.nio.file.Paths; public class SettingsManager { public static final Path SettingsPath = Paths.get(System.getProperty("user.dir"), "settings"); - public static com.chameleonvision.settings.GeneralSettings GeneralSettings; + public static GeneralSettings generalSettings; private SettingsManager() {} public static void initialize() { initGeneralSettings(); var allCameras = CameraManager.getAllCamerasByName(); - if (!allCameras.containsKey(GeneralSettings.currentCamera) && allCameras.size() > 0) { + if (!allCameras.containsKey(generalSettings.currentCamera) && allCameras.size() > 0) { var cam = allCameras.entrySet().stream().findFirst().get().getValue(); - GeneralSettings.currentCamera = cam.name; - GeneralSettings.currentPipeline = cam.getCurrentPipelineIndex(); + generalSettings.currentCamera = cam.name; + generalSettings.currentPipeline = cam.getCurrentPipelineIndex(); } } private static void initGeneralSettings() { FileHelper.CheckPath(SettingsPath); try { - GeneralSettings = new Gson().fromJson(new FileReader(Paths.get(SettingsPath.toString(), "settings.json").toString()), com.chameleonvision.settings.GeneralSettings.class); + generalSettings = new Gson().fromJson(new FileReader(Paths.get(SettingsPath.toString(), "settings.json").toString()), com.chameleonvision.settings.GeneralSettings.class); } catch (FileNotFoundException e) { - GeneralSettings = new GeneralSettings(); + generalSettings = new GeneralSettings(); } } public static void updateCameraSetting(String cameraName, int pipelineNumber) { - GeneralSettings.currentCamera = cameraName; - GeneralSettings.currentPipeline = pipelineNumber; + generalSettings.currentCamera = cameraName; + generalSettings.currentPipeline = pipelineNumber; } public static void updatePipelineSetting(int pipelineNumber) { - GeneralSettings.currentPipeline = pipelineNumber; + generalSettings.currentPipeline = pipelineNumber; } public static void saveSettings() { @@ -55,7 +55,7 @@ public class SettingsManager { try { Gson gson = new GsonBuilder().setPrettyPrinting().create(); FileWriter writer = new FileWriter(Paths.get(SettingsPath.toString(), "settings.json").toString()); - gson.toJson(GeneralSettings, writer); + gson.toJson(generalSettings, writer); writer.flush(); writer.close(); } catch (IOException e) { diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java index 88152ca3e..50bf15ecd 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java @@ -4,19 +4,16 @@ import com.chameleonvision.vision.Pipeline; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.type.MapType; -import com.fasterxml.jackson.databind.type.ArrayType; import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.gson.*; import java.lang.reflect.Type; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -public class CameraDeserializer implements JsonDeserializer { +public class CameraDeserializer implements JsonDeserializer { @Override - public Camera deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context) throws JsonParseException { + public USBCamera deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context) throws JsonParseException { try { var jsonObj = jsonElement.getAsJsonObject(); var camFOV = jsonObj.get("FOV").getAsDouble(); @@ -32,8 +29,8 @@ public class CameraDeserializer implements JsonDeserializer { // always null-check new features boolean isDriver = isDriverObj != null && isDriverObj.getAsBoolean(); - int driverExposure = driverExposureObj == null ? Camera.DEFAULT_EXPOSURE : driverExposureObj.getAsInt(); - int driverBrightness = driverBrightnessObj == null ? Camera.DEFAULT_BRIGHTNESS : driverBrightnessObj.getAsInt(); + int driverExposure = driverExposureObj == null ? USBCamera.DEFAULT_EXPOSURE : driverExposureObj.getAsInt(); + int driverBrightness = driverBrightnessObj == null ? USBCamera.DEFAULT_BRIGHTNESS : driverBrightnessObj.getAsInt(); StreamDivisor divisor = divisorObj == null ? StreamDivisor.none : StreamDivisor.values()[divisorObj.getAsInt()]; var pipelines = jsonObj.get("pipelines"); @@ -47,7 +44,7 @@ public class CameraDeserializer implements JsonDeserializer { e.printStackTrace(); } - var newCamera = actualPipelines != null ? new Camera(camName, camFOV, actualPipelines, videoModeIndex, divisor, isDriver) : new Camera(camName, camFOV, videoModeIndex, divisor, isDriver); + var newCamera = actualPipelines != null ? new USBCamera(camName, camFOV, actualPipelines, videoModeIndex, divisor, isDriver) : new USBCamera(camName, camFOV, videoModeIndex, divisor, isDriver); newCamera.setNickname(camNickname != null ? camNickname : ""); newCamera.setDriverExposure(driverExposure); newCamera.setDriverBrightness(driverBrightness); diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java index 1fcdd9f99..a4855feec 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java @@ -1,9 +1,9 @@ package com.chameleonvision.vision.camera; -import com.chameleonvision.settings.GeneralSettings; import com.chameleonvision.util.FileHelper; import com.chameleonvision.settings.SettingsManager; import com.chameleonvision.vision.Pipeline; +import com.chameleonvision.vision.process.USBCameraProcess; import com.chameleonvision.vision.process.VisionProcess; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -21,10 +21,10 @@ public class CameraManager { private static final Path CamConfigPath = Paths.get(SettingsManager.SettingsPath.toString(), "cameras"); - private static LinkedHashMap AllCamerasByName = new LinkedHashMap<>(); - public static HashMap AllVisionProcessesByName = new HashMap<>(); + private static LinkedHashMap allCamerasByName = new LinkedHashMap<>(); + public static HashMap allVisionProcessesByName = new HashMap<>(); - static HashMap AllUsbCameraInfosByName = new HashMap<>() {{ + static HashMap allUsbCameraInfosByName = new HashMap<>() {{ var suffix = 0; for (var info : UsbCamera.enumerateUsbCameras()) { var cap = new VideoCapture(info.dev); @@ -40,31 +40,31 @@ public class CameraManager { } }}; - public static HashMap getAllCamerasByName() { - return AllCamerasByName; + public static HashMap getAllCamerasByName() { + return allCamerasByName; } public static List getAllCameraByNickname(){ var cameras = getAllCamerasByName(); - return cameras.values().stream().map(Camera::getNickname).collect(Collectors.toList()); + return cameras.values().stream().map(USBCamera::getNickname).collect(Collectors.toList()); } public static boolean initializeCameras() { - if (AllUsbCameraInfosByName.size() == 0) return false; + if (allUsbCameraInfosByName.size() == 0) return false; FileHelper.CheckPath(CamConfigPath); - AllUsbCameraInfosByName.forEach((key, value) -> { + allUsbCameraInfosByName.forEach((key, value) -> { var camPath = Paths.get(CamConfigPath.toString(), String.format("%s.json", key)); File camJsonFile = new File(camPath.toString()); if (camJsonFile.exists() && camJsonFile.length() != 0) { try { - Gson gson = new GsonBuilder().registerTypeAdapter(Camera.class, new CameraDeserializer()).create(); + Gson gson = new GsonBuilder().registerTypeAdapter(USBCamera.class, new CameraDeserializer()).create(); var camJsonFileReader = new FileReader(camPath.toString()); - var gsonRead = gson.fromJson(camJsonFileReader, Camera.class); - AllCamerasByName.put(key, gsonRead); + var gsonRead = gson.fromJson(camJsonFileReader, USBCamera.class); + allCamerasByName.put(key, gsonRead); } catch (FileNotFoundException ex) { ex.printStackTrace(); } } else { - if (!addCamera(new Camera(key), key)) { + if (!addCamera(new USBCamera(key), key)) { System.err.println("Failed to add camera! Already exists!"); } } @@ -73,39 +73,39 @@ public class CameraManager { } public static void initializeThreads(){ - AllCamerasByName.forEach((key, value) -> { - VisionProcess visionProcess = new VisionProcess(value); - AllVisionProcessesByName.put(key, visionProcess); + allCamerasByName.forEach((name, camera) -> { + VisionProcess visionProcess = new VisionProcess(new USBCameraProcess(camera)); + allVisionProcessesByName.put(name, visionProcess); new Thread(visionProcess).start(); }); } - private static boolean addCamera(Camera camera, String cameraName) { - if (AllCamerasByName.containsKey(cameraName)) return false; - camera.addPipeline(); - AllCamerasByName.put(cameraName, camera); + private static boolean addCamera(USBCamera USBCamera, String cameraName) { + if (allCamerasByName.containsKey(cameraName)) return false; + USBCamera.addPipeline(); + allCamerasByName.put(cameraName, USBCamera); return true; } - private static Camera getCamera(String cameraName) { - return AllCamerasByName.get(cameraName); + private static USBCamera getCamera(String cameraName) { + return allCamerasByName.get(cameraName); } - public static Camera getCameraByIndex(int index) { - return AllCamerasByName.get( (AllCamerasByName.keySet().toArray())[ index ] ); + public static USBCamera getCameraByIndex(int index) { + return allCamerasByName.get( (allCamerasByName.keySet().toArray())[ index ] ); } - public static Camera getCurrentCamera() throws CameraException { - if (AllCamerasByName.size() == 0) throw new CameraException(CameraException.CameraExceptionType.NO_CAMERA); - var curCam = AllCamerasByName.get(SettingsManager.GeneralSettings.currentCamera); + public static USBCamera getCurrentCamera() throws CameraException { + if (allCamerasByName.size() == 0) throw new CameraException(CameraException.CameraExceptionType.NO_CAMERA); + var curCam = allCamerasByName.get(SettingsManager.generalSettings.currentCamera); if (curCam == null) throw new CameraException(CameraException.CameraExceptionType.BAD_CAMERA); return curCam; } public static Integer getCurrentCameraIndex() throws CameraException { - if (AllCamerasByName.size() == 0) throw new CameraException(CameraException.CameraExceptionType.NO_CAMERA); - List arr = new ArrayList<>(AllCamerasByName.keySet()); - for (var i = 0; i < AllCamerasByName.size(); i++){ - if (SettingsManager.GeneralSettings.currentCamera.equals(arr.get(i))){ + if (allCamerasByName.size() == 0) throw new CameraException(CameraException.CameraExceptionType.NO_CAMERA); + List arr = new ArrayList<>(allCamerasByName.keySet()); + for (var i = 0; i < allCamerasByName.size(); i++){ + if (SettingsManager.generalSettings.currentCamera.equals(arr.get(i))){ return i; } } @@ -113,13 +113,13 @@ public class CameraManager { } public static void setCurrentCamera(String cameraName) throws CameraException { - if (!AllCamerasByName.containsKey(cameraName)) + if (!allCamerasByName.containsKey(cameraName)) throw new CameraException(CameraException.CameraExceptionType.BAD_CAMERA); - SettingsManager.GeneralSettings.currentCamera = cameraName; + SettingsManager.generalSettings.currentCamera = cameraName; SettingsManager.updateCameraSetting(cameraName, getCurrentCamera().getCurrentPipelineIndex()); } public static void setCurrentCamera(int cameraIndex) throws CameraException { - List s = new ArrayList(AllCamerasByName.keySet()); + List s = new ArrayList(allCamerasByName.keySet()); setCurrentCamera(s.get(cameraIndex)); } @@ -136,20 +136,20 @@ public class CameraManager { } public static VisionProcess getVisionProcessByCameraName(String cameraName) { - return AllVisionProcessesByName.get(cameraName); + return allVisionProcessesByName.get(cameraName); } public static VisionProcess getCurrentVisionProcess() throws CameraException { - if (!SettingsManager.GeneralSettings.currentCamera.equals("")){ - return AllVisionProcessesByName.get(SettingsManager.GeneralSettings.currentCamera); + if (!SettingsManager.generalSettings.currentCamera.equals("")){ + return allVisionProcessesByName.get(SettingsManager.generalSettings.currentCamera); } throw new CameraException(CameraException.CameraExceptionType.NO_CAMERA); } public static void saveCameras() { - for (var entry : AllCamerasByName.entrySet()) { + for (var entry : allCamerasByName.entrySet()) { try { - Gson gson = new GsonBuilder().setPrettyPrinting().registerTypeAdapter(Camera.class, new CameraSerializer()).create(); + Gson gson = new GsonBuilder().setPrettyPrinting().registerTypeAdapter(USBCamera.class, new CameraSerializer()).create(); FileWriter writer = new FileWriter(Paths.get(CamConfigPath.toString(), String.format("%s.json", entry.getKey())).toString()); gson.toJson(entry.getValue(), writer); writer.flush(); diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraSerializer.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraSerializer.java index 313959ddb..8d590dcff 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraSerializer.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraSerializer.java @@ -3,22 +3,22 @@ import com.google.gson.*; import java.lang.reflect.Type; -public class CameraSerializer implements JsonSerializer { +public class CameraSerializer implements JsonSerializer { @Override - public JsonElement serialize(Camera camera, Type type, JsonSerializationContext context) { + public JsonElement serialize(USBCamera USBCamera, Type type, JsonSerializationContext context) { JsonObject obj = new JsonObject(); - obj.addProperty("FOV", camera.getFOV()); - obj.addProperty("path", camera.path); - obj.addProperty("name", camera.name); - obj.addProperty("nickname", camera.getNickname()); - obj.addProperty("streamDivisor", camera.getStreamDivisor().ordinal()); - var pipelines = context.serialize(camera.getPipelines()); + obj.addProperty("FOV", USBCamera.getFOV()); + obj.addProperty("path", USBCamera.path); + obj.addProperty("name", USBCamera.name); + obj.addProperty("nickname", USBCamera.getNickname()); + obj.addProperty("streamDivisor", USBCamera.getStreamDivisor().ordinal()); + var pipelines = context.serialize(USBCamera.getPipelines()); obj.add("pipelines", pipelines); - obj.addProperty("resolution", camera.getVideoModeIndex()); - obj.add("camVideoMode", context.serialize(camera.getVideoMode())); - obj.add("isDriver",context.serialize(camera.getDriverMode())); - obj.add("driverExposure",context.serialize(camera.getDriverExposure())); - obj.add("driverBrightness",context.serialize(camera.getDriverBrightness())); + obj.addProperty("resolution", USBCamera.getVideoModeIndex()); + obj.add("camVideoMode", context.serialize(USBCamera.getVideoMode())); + obj.add("isDriver",context.serialize(USBCamera.getDriverMode())); + obj.add("driverExposure",context.serialize(USBCamera.getDriverExposure())); + obj.add("driverBrightness",context.serialize(USBCamera.getDriverBrightness())); return obj; } } diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java index f429b98d2..50dcde294 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java @@ -21,8 +21,8 @@ public class CameraValues { public final double HorizontalFocalLength; public final double VerticalFocalLength; - public CameraValues(Camera camera) { - this(camera.getVideoMode().width, camera.getVideoMode().height, camera.getFOV()); + public CameraValues(USBCamera USBCamera) { + this(USBCamera.getVideoMode().width, USBCamera.getVideoMode().height, USBCamera.getFOV()); } public CameraValues(int imageWidth, int imageHeight, double fov) { diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/Camera.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCamera.java similarity index 89% rename from Main/src/main/java/com/chameleonvision/vision/camera/Camera.java rename to Main/src/main/java/com/chameleonvision/vision/camera/USBCamera.java index a1ecd7f2a..193947b03 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/Camera.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCamera.java @@ -13,7 +13,7 @@ import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; -public class Camera { +public class USBCamera { private static final double DEFAULT_FOV = 60.8; private static final StreamDivisor DEFAULT_STREAMDIVISOR = StreamDivisor.none; @@ -49,31 +49,31 @@ public class Camera { private int driverBrightness; private boolean isDriver; - public Camera(String cameraName) { + public USBCamera(String cameraName) { this(cameraName, DEFAULT_FOV); } - public Camera(String cameraName, double fov) { - this(cameraName, CameraManager.AllUsbCameraInfosByName.get(cameraName), fov); + public USBCamera(String cameraName, double fov) { + this(cameraName, CameraManager.allUsbCameraInfosByName.get(cameraName), fov); } - public Camera(String cameraName, UsbCameraInfo usbCameraInfo, double fov) { + public USBCamera(String cameraName, UsbCameraInfo usbCameraInfo, double fov) { this(cameraName, usbCameraInfo, fov, DEFAULT_STREAMDIVISOR); } - public Camera(String cameraName, UsbCameraInfo usbCamInfo, double fov, StreamDivisor divisor) { + public USBCamera(String cameraName, UsbCameraInfo usbCamInfo, double fov, StreamDivisor divisor) { this(cameraName, usbCamInfo, fov, new ArrayList<>(), 0, divisor, false); } - public Camera(String cameraName, double fov, List pipelines, int videoModeIndex, StreamDivisor divisor, boolean isDriver) { - this(cameraName, CameraManager.AllUsbCameraInfosByName.get(cameraName), fov, pipelines, videoModeIndex, divisor, isDriver); + public USBCamera(String cameraName, double fov, List pipelines, int videoModeIndex, StreamDivisor divisor, boolean isDriver) { + this(cameraName, CameraManager.allUsbCameraInfosByName.get(cameraName), fov, pipelines, videoModeIndex, divisor, isDriver); } - public Camera(String cameraName, double fov, int videoModeIndex, StreamDivisor divisor, boolean isDriver) { + public USBCamera(String cameraName, double fov, int videoModeIndex, StreamDivisor divisor, boolean isDriver) { this(cameraName, fov, new ArrayList<>(), videoModeIndex, divisor, isDriver); } - public Camera(String cameraName, UsbCameraInfo usbCamInfo, double fov, List pipelines, int videoModeIndex, StreamDivisor divisor, boolean isDriver) { + public USBCamera(String cameraName, UsbCameraInfo usbCamInfo, double fov, List pipelines, int videoModeIndex, StreamDivisor divisor, boolean isDriver) { FOV = fov; name = cameraName; @@ -99,13 +99,13 @@ public class Camera { } } var initTimeMs = (System.nanoTime() - initTimeout) / 1e6; - System.out.printf("Camera initialized in %.2fms\n", initTimeMs); + System.out.printf("USBCamera initialized in %.2fms\n", initTimeMs); } var trueVideoModes = UsbCam.enumerateVideoModes(); availableVideoModes = Arrays.stream(trueVideoModes).filter(v -> v.fps >= MINIMUM_FPS && v.width >= MINIMUM_WIDTH && v.height >= MINIMUM_HEIGHT && ALLOWED_PIXEL_FORMATS.contains(v.pixelFormat)).toArray(VideoMode[]::new); if (availableVideoModes.length == 0) { - System.err.println("Camera not supported!"); + System.err.println("USBCamera not supported!"); throw new RuntimeException(new CameraException(CameraException.CameraExceptionType.BAD_CAMERA)); } if (videoModeIndex <= availableVideoModes.length - 1) { @@ -283,7 +283,7 @@ public class Camera { try { UsbCam.setExposureManual(exposure); } catch (VideoException e) { - System.err.println("Camera Does not support exposure change"); + System.err.println("USBCamera Does not support exposure change"); } } @@ -311,9 +311,9 @@ public class Camera { //Deletes old camera nt table NetworkTableInstance.getDefault().getTable("/chameleon-vision/" + this.nickname).getInstance().deleteAllEntries(); nickname = newNickname; - if (CameraManager.AllVisionProcessesByName.containsKey(this.name)) { + if (CameraManager.allVisionProcessesByName.containsKey(this.name)) { NetworkTable newNT = NetworkTableInstance.getDefault().getTable("/chameleon-vision/" + this.nickname); - CameraManager.AllVisionProcessesByName.get(this.name).resetNT(newNT); + CameraManager.allVisionProcessesByName.get(this.name).resetNT(newNT); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java index 0f9405cf3..398274a1a 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java @@ -1,77 +1,31 @@ package com.chameleonvision.vision.process; -import com.chameleonvision.vision.camera.Camera; -import com.chameleonvision.vision.camera.StreamDivisor; -import org.opencv.core.CvType; +import com.chameleonvision.vision.Pipeline; +import com.chameleonvision.vision.camera.CamVideoMode; +import com.chameleonvision.vision.camera.CameraValues; import org.opencv.core.Mat; -import org.opencv.core.Size; -import org.opencv.imgproc.Imgproc; -public class CameraProcess implements Runnable { +import java.util.List; - private final Camera camera; - private final int maxFPS; - private final Object inputFrameLock = new Object(); - private final Object outputFrameLock = new Object(); - private Mat inputFrame; - private Mat outputFrame; - private long timestamp; - private StreamDivisor divisor; +public interface CameraProcess extends Runnable { - CameraProcess(Camera camera) { - this.camera = camera; - maxFPS = camera.getVideoMode().fps; - updateFrameSize(); - } + long getLatestFrame(Mat outputFrame); - public void updateFrameSize() { - var camVals = camera.getCamVals(); - divisor = camera.getStreamDivisor(); - var newWidth = camVals.ImageWidth / divisor.value; - var newHeight = camVals.ImageHeight / divisor.value; - synchronized (inputFrameLock) { - inputFrame = new Mat(newWidth, newHeight, CvType.CV_8UC3); - } - synchronized (outputFrameLock) { - outputFrame = new Mat(newWidth, newHeight, CvType.CV_8UC3); - } - } + void updateFrame(Mat inputFrame); - void updateFrame(Mat inputFrame) { - synchronized (inputFrameLock) { - inputFrame.copyTo(this.inputFrame); - } - } + void updateFrameSize(); - long getLatestFrame(Mat outputFrame) { - synchronized (outputFrameLock) { - this.outputFrame.copyTo(outputFrame); - return timestamp; - } - } + String getCamName(); + CameraValues getCamVals(); + boolean getDriverMode(); + void setDriverMode(boolean isDriverMode); + List getPipelines(); + Pipeline getCurrentPipeline(); + int getCurrentPipelineIndex(); + void setExposure(int exposure); + void setBrightness(int brightness); + CamVideoMode getVideoMode(); + String getNickname(); + void setCurrentPipelineIndex(int ntPipelineIndex); - @Override - public void run() { - while (!Thread.interrupted()) { - synchronized (outputFrameLock) { - timestamp = camera.grabFrame(outputFrame); - } - synchronized (inputFrameLock) { - if (divisor.value != 1) { - var camVals = camera.getCamVals(); - var newWidth = camVals.ImageWidth / divisor.value; - var newHeight = camVals.ImageHeight / divisor.value; - Size newSize = new Size(newWidth, newHeight); - Imgproc.resize(inputFrame, inputFrame, newSize); - } - camera.putFrame(inputFrame); - } - var msToWait = (long) 1000 / maxFPS; - try { - Thread.sleep(msToWait); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java new file mode 100644 index 000000000..3491c893a --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java @@ -0,0 +1,144 @@ +package com.chameleonvision.vision.process; + +import com.chameleonvision.vision.Pipeline; +import com.chameleonvision.vision.camera.CamVideoMode; +import com.chameleonvision.vision.camera.CameraValues; +import com.chameleonvision.vision.camera.USBCamera; +import com.chameleonvision.vision.camera.StreamDivisor; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; + +import java.util.List; + +public class USBCameraProcess implements CameraProcess { + + private final USBCamera usbCamera; + private final int maxFPS; + private final Object inputFrameLock = new Object(); + private final Object outputFrameLock = new Object(); + private Mat inputFrame; + private Mat outputFrame; + private long timestamp; + private StreamDivisor divisor; + + public USBCameraProcess(USBCamera usbCamera) { + this.usbCamera = usbCamera; + maxFPS = usbCamera.getVideoMode().fps; + updateFrameSize(); + } + + public void updateFrameSize() { + var camVals = usbCamera.getCamVals(); + divisor = usbCamera.getStreamDivisor(); + var newWidth = camVals.ImageWidth / divisor.value; + var newHeight = camVals.ImageHeight / divisor.value; + synchronized (inputFrameLock) { + inputFrame = new Mat(newWidth, newHeight, CvType.CV_8UC3); + } + synchronized (outputFrameLock) { + outputFrame = new Mat(newWidth, newHeight, CvType.CV_8UC3); + } + } + + public void updateFrame(Mat inputFrame) { + synchronized (inputFrameLock) { + inputFrame.copyTo(this.inputFrame); + } + } + + public long getLatestFrame(Mat outputFrame) { + synchronized (outputFrameLock) { + this.outputFrame.copyTo(outputFrame); + return timestamp; + } + } + + @Override + public void run() { + while (!Thread.interrupted()) { + synchronized (outputFrameLock) { + timestamp = usbCamera.grabFrame(outputFrame); + } + synchronized (inputFrameLock) { + if (divisor.value != 1) { + var camVals = usbCamera.getCamVals(); + var newWidth = camVals.ImageWidth / divisor.value; + var newHeight = camVals.ImageHeight / divisor.value; + Size newSize = new Size(newWidth, newHeight); + Imgproc.resize(inputFrame, inputFrame, newSize); + } + usbCamera.putFrame(inputFrame); + } + var msToWait = (long) 1000 / maxFPS; + try { + Thread.sleep(msToWait); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + // USBCamera stuff + + @Override + public String getCamName() { + return usbCamera.name; + } + + @Override + public CameraValues getCamVals() { + return usbCamera.getCamVals(); + } + + @Override + public boolean getDriverMode() { + return usbCamera.getDriverMode(); + } + + @Override + public void setDriverMode(boolean isDriverMode) { + usbCamera.setDriverMode(isDriverMode); + } + + @Override + public List getPipelines() { + return usbCamera.getPipelines(); + } + + @Override + public Pipeline getCurrentPipeline() { + return usbCamera.getCurrentPipeline(); + } + + @Override + public int getCurrentPipelineIndex() { + return usbCamera.getCurrentPipelineIndex(); + } + + @Override + public void setExposure(int exposure) { + usbCamera.setExposure(exposure); + } + + @Override + public void setBrightness(int brightness) { + usbCamera.setBrightness(brightness); + } + + @Override + public CamVideoMode getVideoMode() { + return usbCamera.getVideoMode(); + } + + @Override + public String getNickname() { + return usbCamera.getNickname(); + } + + @Override + public void setCurrentPipelineIndex(int wantedIndex) { + usbCamera.setCurrentPipelineIndex(wantedIndex); + } +} diff --git a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java index 6e00b6477..ce7ac6e04 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java @@ -1,10 +1,8 @@ package com.chameleonvision.vision.process; import com.chameleonvision.settings.SettingsManager; -import com.chameleonvision.vision.CalibrationMode; import com.chameleonvision.vision.Orientation; import com.chameleonvision.vision.Pipeline; -import com.chameleonvision.vision.camera.Camera; import com.chameleonvision.web.ServerHandler; import edu.wpi.cscore.VideoException; import edu.wpi.first.networktables.*; @@ -17,7 +15,6 @@ import java.util.List; public class VisionProcess implements Runnable { - private final Camera camera; private final String cameraName; public final CameraProcess cameraProcess; // NetworkTables @@ -45,36 +42,36 @@ public class VisionProcess implements Runnable { private Scalar BoxRectColor = new Scalar(0, 0, 233); private long timeStamp = 0; - public VisionProcess(Camera processCam) { - camera = processCam; - this.cameraName = camera.name; + public VisionProcess(CameraProcess cameraProcess) { - initNT(NetworkTableInstance.getDefault().getTable("/chameleon-vision/"+processCam.getNickname())); + // USBCamera settings + cvProcess = new CVProcess(cameraProcess.getCamVals()); + this.cameraProcess = cameraProcess; // new USBCameraProcess(cameraProcess); - // camera settings - cvProcess = new CVProcess(camera.getCamVals()); - cameraProcess = new CameraProcess(camera); + this.cameraName = cameraProcess.getCamName(); + + initNT(NetworkTableInstance.getDefault().getTable("/chameleon-vision/" + cameraProcess.getNickname())); } private void driverModeListener(EntryNotification entryNotification) { - camera.setDriverMode(entryNotification.value.getBoolean()); + cameraProcess.setDriverMode(entryNotification.value.getBoolean()); } private void pipelineListener(EntryNotification entryNotification) { var ntPipelineIndex = (int) entryNotification.value.getDouble(); - if (ntPipelineIndex >= camera.getPipelines().size()) { - ntPipelineEntry.setNumber(camera.getCurrentPipelineIndex()); + if (ntPipelineIndex >= cameraProcess.getPipelines().size()) { + ntPipelineEntry.setNumber(cameraProcess.getCurrentPipelineIndex()); } else { - var pipeline = camera.getCurrentPipeline(); - camera.setCurrentPipelineIndex(ntPipelineIndex); + var pipeline = cameraProcess.getCurrentPipeline(); + cameraProcess.setCurrentPipelineIndex(ntPipelineIndex); try { - camera.setExposure(pipeline.exposure); + cameraProcess.setExposure(pipeline.exposure); } catch (VideoException e) { System.err.println(e.toString()); } - camera.setBrightness(pipeline.brightness); - if (SettingsManager.GeneralSettings.currentCamera.equals(cameraName)) { - SettingsManager.GeneralSettings.currentPipeline = ntPipelineIndex; + cameraProcess.setBrightness(pipeline.brightness); + if (SettingsManager.generalSettings.currentCamera.equals(cameraName)) { + SettingsManager.generalSettings.currentPipeline = ntPipelineIndex; HashMap pipeChange = new HashMap<>(); pipeChange.put("currentPipeline", ntPipelineIndex); ServerHandler.broadcastMessage(pipeChange); @@ -123,7 +120,7 @@ public class VisionProcess implements Runnable { if (currentPipeline.orientation.equals(Orientation.Inverted)) { Core.flip(inputImage, inputImage, -1); } - if (camera.getDriverMode()) { + if (cameraProcess.getDriverMode()) { inputImage.copyTo(outputImage); return pipelineResult; } @@ -150,9 +147,9 @@ public class VisionProcess implements Runnable { pipelineResult.IsValid = true; switch (currentPipeline.calibrationMode) { case None: - ///use the center of the camera to find the pitch and yaw difference - pipelineResult.CalibratedX = camera.getCamVals().CenterX; - pipelineResult.CalibratedY = camera.getCamVals().CenterY; + ///use the center of the USBCamera to find the pitch and yaw difference + pipelineResult.CalibratedX = cameraProcess.getCamVals().CenterX; + pipelineResult.CalibratedY = cameraProcess.getCamVals().CenterY; break; case Single: // use the static point as a calibration method instead of the center @@ -165,7 +162,7 @@ public class VisionProcess implements Runnable { pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; break; } -// var camVals = camera.getCamVals(); +// var camVals = cameraProcess.getCamVals(); // if (currentPipeline.isCalibrated) { // pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; // pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; @@ -173,8 +170,8 @@ public class VisionProcess implements Runnable { // pipelineResult.CalibratedX = camVals.CenterX; // pipelineResult.CalibratedY = camVals.CenterY; // } - pipelineResult.Pitch = camera.getCamVals().CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); - pipelineResult.Yaw = camera.getCamVals().CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); + pipelineResult.Pitch = cameraProcess.getCamVals().CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); + pipelineResult.Yaw = cameraProcess.getCamVals().CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); pipelineResult.Area = finalRect.size.area(); drawContour(outputImage, finalRect); } @@ -193,7 +190,7 @@ public class VisionProcess implements Runnable { double processTimeMs; double fps = 0; double uiFps = 0; - int maxFps = camera.getVideoMode().fps; + int maxFps = cameraProcess.getVideoMode().fps; new Thread(cameraProcess).start(); @@ -201,10 +198,11 @@ public class VisionProcess implements Runnable { while (!Thread.interrupted()) { startTime = System.nanoTime(); - if ((startTime - lastFrameEndNanosec) * 1e-6 >= 1000.0 / maxFps + 3) { // 3 additional fps to allow for overhead + if ((startTime - lastFrameEndNanosec) * 1e-6 >= 1000.0 / (maxFps + 3)) { // 3 additional fps to allow for overhead foundContours.clear(); filteredContours.clear(); groupedContours.clear(); + deSpeckledContours.clear(); // update FPS for ui only every 0.5 seconds if ((startTime - fpsLastTime) * 1e-6 >= 500) { @@ -216,7 +214,7 @@ public class VisionProcess implements Runnable { fpsLastTime = System.nanoTime(); } - currentPipeline = camera.getCurrentPipeline(); + currentPipeline = cameraProcess.getCurrentPipeline(); // start fps counter right before grabbing input frame timeStamp = cameraProcess.getLatestFrame(cameraInputMat); if (cameraInputMat.cols() == 0 && cameraInputMat.rows() == 0) { @@ -226,7 +224,7 @@ public class VisionProcess implements Runnable { // get vision data var pipelineResult = runVisionProcess(cameraInputMat, streamOutputMat); updateNetworkTables(pipelineResult); - if (cameraName.equals(SettingsManager.GeneralSettings.currentCamera)) { + if (cameraName.equals(SettingsManager.generalSettings.currentCamera)) { HashMap WebSend = new HashMap<>(); HashMap point = new HashMap<>(); HashMap calculated = new HashMap<>(); @@ -293,6 +291,6 @@ public class VisionProcess implements Runnable { ntDriveModeListenerID = ntDriverModeEntry.addListener(this::driverModeListener, EntryListenerFlags.kUpdate); ntPipelineListenerID = ntPipelineEntry.addListener(this::pipelineListener, EntryListenerFlags.kUpdate); ntDriverModeEntry.setBoolean(false); - ntPipelineEntry.setNumber(camera.getCurrentPipelineIndex()); + ntPipelineEntry.setNumber(cameraProcess.getCurrentPipelineIndex()); } } diff --git a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java index 214ca6f5b..9a60c716f 100644 --- a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java +++ b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java @@ -1,23 +1,19 @@ package com.chameleonvision.web; -import com.chameleonvision.settings.GeneralSettings; import com.chameleonvision.vision.*; -import com.chameleonvision.vision.camera.Camera; +import com.chameleonvision.vision.camera.USBCamera; import com.chameleonvision.vision.camera.CameraException; import com.chameleonvision.settings.SettingsManager; import com.chameleonvision.vision.camera.CameraManager; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import edu.wpi.cscore.VideoException; import io.javalin.websocket.*; import org.apache.commons.lang3.ArrayUtils; import org.msgpack.jackson.dataformat.MessagePackFactory; -import java.io.IOException; import java.lang.reflect.Field; -import java.lang.reflect.ParameterizedType; import java.nio.ByteBuffer; import java.util.*; @@ -49,7 +45,7 @@ public class ServerHandler { switch (entry.getKey()) { case "generalSettings": { for (HashMap.Entry e : ((HashMap) entry.getValue()).entrySet()) { - setField(SettingsManager.GeneralSettings, e.getKey(), e.getValue()); + setField(SettingsManager.generalSettings, e.getKey(), e.getValue()); } SettingsManager.saveSettings(); sendFullSettings(); @@ -182,8 +178,8 @@ public class ServerHandler { private void setField(Object obj, String fieldName, Object value) { try { - if (obj instanceof Camera) { - var cam = (Camera)obj; + if (obj instanceof USBCamera) { + var cam = (USBCamera)obj; if (fieldName.equals("driverBrightness")) { cam.setDriverBrightness((Integer)value); } else if (fieldName.equals("driverExposure")) { @@ -234,12 +230,12 @@ public class ServerHandler { private static HashMap getOrdinalSettings() { HashMap tmp = new HashMap<>(); - tmp.put("teamNumber", SettingsManager.GeneralSettings.teamNumber); - tmp.put("connectionType", SettingsManager.GeneralSettings.connectionType.ordinal()); - tmp.put("ip", SettingsManager.GeneralSettings.ip); - tmp.put("gateway", SettingsManager.GeneralSettings.gateway); - tmp.put("netmask", SettingsManager.GeneralSettings.netmask); - tmp.put("hostname", SettingsManager.GeneralSettings.hostname); + tmp.put("teamNumber", SettingsManager.generalSettings.teamNumber); + tmp.put("connectionType", SettingsManager.generalSettings.connectionType.ordinal()); + tmp.put("ip", SettingsManager.generalSettings.ip); + tmp.put("gateway", SettingsManager.generalSettings.gateway); + tmp.put("netmask", SettingsManager.generalSettings.netmask); + tmp.put("hostname", SettingsManager.generalSettings.hostname); return tmp; } From e863cc07810262b5a394d8c9204db01a9fbf8434 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 2 Nov 2019 12:38:42 -0700 Subject: [PATCH 02/84] Move pipeline logic into CVProcess --- .../vision/process/CVProcess.java | 99 ++++++++++++++++++- .../vision/process/CameraProcess.java | 3 + .../vision/process/VisionProcess.java | 91 ++--------------- 3 files changed, 107 insertions(+), 86 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java index 51256291a..f99d26657 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java @@ -1,8 +1,6 @@ package com.chameleonvision.vision.process; -import com.chameleonvision.vision.SortMode; -import com.chameleonvision.vision.TargetGroup; -import com.chameleonvision.vision.TargetIntersection; +import com.chameleonvision.vision.*; import com.chameleonvision.vision.camera.CameraValues; import com.chameleonvision.util.MathHandler; import org.apache.commons.math3.util.FastMath; @@ -34,6 +32,101 @@ public class CVProcess { this.cameraValues = cameraValues; } + private Mat cameraInputMat = new Mat(); + private Mat hsvThreshMat = new Mat(); + private Mat streamOutputMat = new Mat(); + + private List foundContours_ = new ArrayList<>(); + private List filteredContours_ = new ArrayList<>(); + private List deSpeckledContours_ = new ArrayList<>(); + private List groupedContours_ = new ArrayList<>(); + + private static final Scalar contourRectColor = new Scalar(255, 0, 0); + private static final Scalar BoxRectColor = new Scalar(0, 0, 233); + + private void drawContour(Mat inputMat, RotatedRect contourRect) { + if (contourRect == null) return; + List drawnContour = new ArrayList<>(); + Point[] vertices = new Point[4]; + contourRect.points(vertices); + MatOfPoint contour = new MatOfPoint(vertices); + drawnContour.add(contour); + Rect box = Imgproc.boundingRect(contour); + Imgproc.drawContours(inputMat, drawnContour, 0, contourRectColor, 3); + Imgproc.circle(inputMat, contourRect.center, 3, contourRectColor); + Imgproc.rectangle(inputMat, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), BoxRectColor, 2); + } + + PipelineResult runPipeline(Pipeline currentPipeline, Mat inputImage, Mat outputImage, CameraValues cameraValues, boolean shouldFlip, boolean driverMode) { + var pipelineResult = new PipelineResult(); + + if (currentPipeline == null) { + return pipelineResult; + } + if (shouldFlip) { + Core.flip(inputImage, inputImage, -1); + } + if (driverMode) { + inputImage.copyTo(outputImage); + return pipelineResult; + } + + foundContours_.clear(); + filteredContours_.clear(); + deSpeckledContours_.clear(); + groupedContours_.clear(); + + Scalar hsvLower = new Scalar(currentPipeline.hue.get(0).intValue(), currentPipeline.saturation.get(0).intValue(), currentPipeline.value.get(0).intValue()); + Scalar hsvUpper = new Scalar(currentPipeline.hue.get(1).intValue(), currentPipeline.saturation.get(1).intValue(), currentPipeline.value.get(1).intValue()); + + hsvThreshold(inputImage, hsvThreshMat, hsvLower, hsvUpper, currentPipeline.erode, currentPipeline.dilate); + + if (currentPipeline.isBinary) { + Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); + } else { + inputImage.copyTo(outputImage); + } + foundContours_ = findContours(hsvThreshMat); + if (foundContours_.size() > 0) { + filteredContours_ = filterContours(foundContours_, currentPipeline.area, currentPipeline.ratio, currentPipeline.extent); + if (filteredContours_.size() > 0) { + deSpeckledContours_ = rejectSpeckles(filteredContours_, currentPipeline.speckle.doubleValue()); + if (deSpeckledContours_.size() > 0) { + groupedContours_ = groupTargets(deSpeckledContours_, currentPipeline.targetIntersection, currentPipeline.targetGroup); + if (groupedContours_.size() > 0) { + var finalRect = sortTargetsToOne(groupedContours_, currentPipeline.sortMode); + pipelineResult.RawPoint = finalRect; + pipelineResult.IsValid = true; + switch (currentPipeline.calibrationMode) { + case None: + ///use the center of the USBCamera to find the pitch and yaw difference + pipelineResult.CalibratedX = cameraValues.CenterX; + pipelineResult.CalibratedY = cameraValues.CenterY; + break; + case Single: + // use the static point as a calibration method instead of the center + pipelineResult.CalibratedX = currentPipeline.point.get(0).doubleValue(); + pipelineResult.CalibratedY = currentPipeline.point.get(1).doubleValue(); + break; + case Dual: + // use the calculated line to find the difference in length between the point and the line + pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; + pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; + break; + } + + pipelineResult.Pitch = cameraValues.CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); + pipelineResult.Yaw = cameraValues.CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); + pipelineResult.Area = finalRect.size.area(); + drawContour(outputImage, finalRect); + } + } + } + } + + return pipelineResult; + } + void hsvThreshold(Mat srcImage, Mat dst, @NotNull Scalar hsvLower, @NotNull Scalar hsvUpper, boolean shouldErode, boolean shouldDilate) { Imgproc.cvtColor(srcImage, hsvImage, Imgproc.COLOR_RGB2HSV, 3); Imgproc.blur(hsvImage, hsvImage, blur); diff --git a/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java index 398274a1a..00582ea50 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java @@ -28,4 +28,7 @@ public interface CameraProcess extends Runnable { String getNickname(); void setCurrentPipelineIndex(int ntPipelineIndex); + PipelineResult runPipeline(Pipeline currentPipeline, Mat inputImage, Mat outputImage, + CameraValues cameraValues, boolean shouldFlip, boolean driverMode); + } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java index ce7ac6e04..eca77787b 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java @@ -38,8 +38,6 @@ public class VisionProcess implements Runnable { private Mat cameraInputMat = new Mat(); private Mat hsvThreshMat = new Mat(); private Mat streamOutputMat = new Mat(); - private Scalar contourRectColor = new Scalar(255, 0, 0); - private Scalar BoxRectColor = new Scalar(0, 0, 233); private long timeStamp = 0; public VisionProcess(CameraProcess cameraProcess) { @@ -81,19 +79,6 @@ public class VisionProcess implements Runnable { } } - private void drawContour(Mat inputMat, RotatedRect contourRect) { - if (contourRect == null) return; - List drawnContour = new ArrayList<>(); - Point[] vertices = new Point[4]; - contourRect.points(vertices); - MatOfPoint contour = new MatOfPoint(vertices); - drawnContour.add(contour); - Rect box = Imgproc.boundingRect(contour); - Imgproc.drawContours(inputMat, drawnContour, 0, contourRectColor, 3); - Imgproc.circle(inputMat, contourRect.center, 3, contourRectColor); - Imgproc.rectangle(inputMat, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), BoxRectColor, 2); - } - private void updateNetworkTables(PipelineResult pipelineResult) { if (pipelineResult.IsValid) { ntValidEntry.setBoolean(true); @@ -112,74 +97,14 @@ public class VisionProcess implements Runnable { } private PipelineResult runVisionProcess(Mat inputImage, Mat outputImage) { - var pipelineResult = new PipelineResult(); - - if (currentPipeline == null) { - return pipelineResult; - } - if (currentPipeline.orientation.equals(Orientation.Inverted)) { - Core.flip(inputImage, inputImage, -1); - } - if (cameraProcess.getDriverMode()) { - inputImage.copyTo(outputImage); - return pipelineResult; - } - Scalar hsvLower = new Scalar(currentPipeline.hue.get(0).intValue(), currentPipeline.saturation.get(0).intValue(), currentPipeline.value.get(0).intValue()); - Scalar hsvUpper = new Scalar(currentPipeline.hue.get(1).intValue(), currentPipeline.saturation.get(1).intValue(), currentPipeline.value.get(1).intValue()); - - cvProcess.hsvThreshold(inputImage, hsvThreshMat, hsvLower, hsvUpper, currentPipeline.erode, currentPipeline.dilate); - - if (currentPipeline.isBinary) { - Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); - } else { - inputImage.copyTo(outputImage); - } - foundContours = cvProcess.findContours(hsvThreshMat); - if (foundContours.size() > 0) { - filteredContours = cvProcess.filterContours(foundContours, currentPipeline.area, currentPipeline.ratio, currentPipeline.extent); - if (filteredContours.size() > 0) { - deSpeckledContours = cvProcess.rejectSpeckles(filteredContours, currentPipeline.speckle.doubleValue()); - if (deSpeckledContours.size() > 0) { - groupedContours = cvProcess.groupTargets(deSpeckledContours, currentPipeline.targetIntersection, currentPipeline.targetGroup); - if (groupedContours.size() > 0) { - var finalRect = cvProcess.sortTargetsToOne(groupedContours, currentPipeline.sortMode); - pipelineResult.RawPoint = finalRect; - pipelineResult.IsValid = true; - switch (currentPipeline.calibrationMode) { - case None: - ///use the center of the USBCamera to find the pitch and yaw difference - pipelineResult.CalibratedX = cameraProcess.getCamVals().CenterX; - pipelineResult.CalibratedY = cameraProcess.getCamVals().CenterY; - break; - case Single: - // use the static point as a calibration method instead of the center - pipelineResult.CalibratedX = currentPipeline.point.get(0).doubleValue(); - pipelineResult.CalibratedY = currentPipeline.point.get(1).doubleValue(); - break; - case Dual: - // use the calculated line to find the difference in length between the point and the line - pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; - pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; - break; - } -// var camVals = cameraProcess.getCamVals(); -// if (currentPipeline.isCalibrated) { -// pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; -// pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; -// } else { -// pipelineResult.CalibratedX = camVals.CenterX; -// pipelineResult.CalibratedY = camVals.CenterY; -// } - pipelineResult.Pitch = cameraProcess.getCamVals().CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); - pipelineResult.Yaw = cameraProcess.getCamVals().CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); - pipelineResult.Area = finalRect.size.area(); - drawContour(outputImage, finalRect); - } - } - } - } - - return pipelineResult; + return cameraProcess.runPipeline( + currentPipeline, + inputImage, + outputImage, + cameraProcess.getCamVals(), + currentPipeline.orientation.equals(Orientation.Inverted), + cameraProcess.getDriverMode() + ); } @Override From e055e5a78b77f7f11607b5e3b11385eb00e4cd8d Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 2 Nov 2019 12:44:05 -0700 Subject: [PATCH 03/84] invert the nested if/else in the run pipeline This should be more clear to read --- .../vision/process/CVProcess.java | 91 +++++++++++-------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java index f99d26657..0e0ce97d6 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java @@ -63,9 +63,13 @@ public class CVProcess { if (currentPipeline == null) { return pipelineResult; } + + // flip the image if (shouldFlip) { Core.flip(inputImage, inputImage, -1); } + + // if we're in driver mode don't do anything, and return a blank result if (driverMode) { inputImage.copyTo(outputImage); return pipelineResult; @@ -76,54 +80,69 @@ public class CVProcess { deSpeckledContours_.clear(); groupedContours_.clear(); + // HSV threshold the image Scalar hsvLower = new Scalar(currentPipeline.hue.get(0).intValue(), currentPipeline.saturation.get(0).intValue(), currentPipeline.value.get(0).intValue()); Scalar hsvUpper = new Scalar(currentPipeline.hue.get(1).intValue(), currentPipeline.saturation.get(1).intValue(), currentPipeline.value.get(1).intValue()); - hsvThreshold(inputImage, hsvThreshMat, hsvLower, hsvUpper, currentPipeline.erode, currentPipeline.dilate); + // Make sure we're BFR if (currentPipeline.isBinary) { Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); } else { inputImage.copyTo(outputImage); } + + // search for contours foundContours_ = findContours(hsvThreshMat); - if (foundContours_.size() > 0) { - filteredContours_ = filterContours(foundContours_, currentPipeline.area, currentPipeline.ratio, currentPipeline.extent); - if (filteredContours_.size() > 0) { - deSpeckledContours_ = rejectSpeckles(filteredContours_, currentPipeline.speckle.doubleValue()); - if (deSpeckledContours_.size() > 0) { - groupedContours_ = groupTargets(deSpeckledContours_, currentPipeline.targetIntersection, currentPipeline.targetGroup); - if (groupedContours_.size() > 0) { - var finalRect = sortTargetsToOne(groupedContours_, currentPipeline.sortMode); - pipelineResult.RawPoint = finalRect; - pipelineResult.IsValid = true; - switch (currentPipeline.calibrationMode) { - case None: - ///use the center of the USBCamera to find the pitch and yaw difference - pipelineResult.CalibratedX = cameraValues.CenterX; - pipelineResult.CalibratedY = cameraValues.CenterY; - break; - case Single: - // use the static point as a calibration method instead of the center - pipelineResult.CalibratedX = currentPipeline.point.get(0).doubleValue(); - pipelineResult.CalibratedY = currentPipeline.point.get(1).doubleValue(); - break; - case Dual: - // use the calculated line to find the difference in length between the point and the line - pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; - pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; - break; - } - - pipelineResult.Pitch = cameraValues.CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); - pipelineResult.Yaw = cameraValues.CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); - pipelineResult.Area = finalRect.size.area(); - drawContour(outputImage, finalRect); - } - } - } + if (foundContours_.size() < 1) { + return pipelineResult; } + // filter contours by area, ratio and extent + filteredContours_ = filterContours(foundContours_, currentPipeline.area, currentPipeline.ratio, currentPipeline.extent); + if (filteredContours_.size() < 1) { + return pipelineResult; + } + + // reject "speckle" contours + deSpeckledContours_ = rejectSpeckles(filteredContours_, currentPipeline.speckle.doubleValue()); + if (deSpeckledContours_.size() < 1) { + return pipelineResult; + } + + // group targets + groupedContours_ = groupTargets(deSpeckledContours_, currentPipeline.targetIntersection, currentPipeline.targetGroup); + if (groupedContours_.size() < 1) { + return pipelineResult; + } + + // sort targets down to our final target + var finalRect = sortTargetsToOne(groupedContours_, currentPipeline.sortMode); + pipelineResult.RawPoint = finalRect; + pipelineResult.IsValid = true; + switch (currentPipeline.calibrationMode) { + case None: + ///use the center of the USBCamera to find the pitch and yaw difference + pipelineResult.CalibratedX = cameraValues.CenterX; + pipelineResult.CalibratedY = cameraValues.CenterY; + break; + case Single: + // use the static point as a calibration method instead of the center + pipelineResult.CalibratedX = currentPipeline.point.get(0).doubleValue(); + pipelineResult.CalibratedY = currentPipeline.point.get(1).doubleValue(); + break; + case Dual: + // use the calculated line to find the difference in length between the point and the line + pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; + pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; + break; + } + + pipelineResult.Pitch = cameraValues.CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); + pipelineResult.Yaw = cameraValues.CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); + pipelineResult.Area = finalRect.size.area(); + drawContour(outputImage, finalRect); + return pipelineResult; } From 9f68be992202d5c07a903ff28b7ef2f0369c7087 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 2 Nov 2019 12:46:36 -0700 Subject: [PATCH 04/84] Fix cameraProcess/cvProcess missmatch --- .../com/chameleonvision/vision/process/CameraProcess.java | 4 ---- .../com/chameleonvision/vision/process/VisionProcess.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java index 00582ea50..9683c00a6 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/CameraProcess.java @@ -27,8 +27,4 @@ public interface CameraProcess extends Runnable { CamVideoMode getVideoMode(); String getNickname(); void setCurrentPipelineIndex(int ntPipelineIndex); - - PipelineResult runPipeline(Pipeline currentPipeline, Mat inputImage, Mat outputImage, - CameraValues cameraValues, boolean shouldFlip, boolean driverMode); - } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java index eca77787b..4c5b234de 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java @@ -97,7 +97,7 @@ public class VisionProcess implements Runnable { } private PipelineResult runVisionProcess(Mat inputImage, Mat outputImage) { - return cameraProcess.runPipeline( + return cvProcess.runPipeline( currentPipeline, inputImage, outputImage, From 42b76dfbf9cb414d3f36b96020d8fc9b1ce93757 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 2 Nov 2019 13:03:00 -0700 Subject: [PATCH 05/84] Rename CVProcess to StandardCVProcess --- .../vision/process/CVProcess.java | 344 +---------------- .../vision/process/StandardCVProcess.java | 345 ++++++++++++++++++ .../vision/process/VisionProcess.java | 19 +- 3 files changed, 358 insertions(+), 350 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java diff --git a/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java index 0e0ce97d6..5105fa535 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/CVProcess.java @@ -1,345 +1,9 @@ package com.chameleonvision.vision.process; -import com.chameleonvision.vision.*; +import com.chameleonvision.vision.Pipeline; import com.chameleonvision.vision.camera.CameraValues; -import com.chameleonvision.util.MathHandler; -import org.apache.commons.math3.util.FastMath; -import org.jetbrains.annotations.NotNull; -import org.opencv.core.*; -import org.opencv.imgproc.Imgproc; -import org.opencv.imgproc.Moments; +import org.opencv.core.Mat; -import java.util.*; - -@SuppressWarnings("WeakerAccess") -public class CVProcess { - - private final CameraValues cameraValues; - private Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5, 5)); - private Size blur = new Size(3, 3); - private Mat hsvImage = new Mat(); - private List foundContours = new ArrayList<>(); - private Mat binaryMat = new Mat(); - private List filteredContours = new ArrayList<>(); - private Comparator sortByCentermostComparator = Comparator.comparingDouble(this::calcDistance); - private List speckleRejectedContours = new ArrayList<>(); - private Comparator sortByMomentsX = Comparator.comparingDouble(this::calcMomentsX); - private List finalCountours = new ArrayList<>(); - private MatOfPoint2f intersectMatA = new MatOfPoint2f(); - private MatOfPoint2f intersectMatB = new MatOfPoint2f(); - - CVProcess(CameraValues cameraValues) { - this.cameraValues = cameraValues; - } - - private Mat cameraInputMat = new Mat(); - private Mat hsvThreshMat = new Mat(); - private Mat streamOutputMat = new Mat(); - - private List foundContours_ = new ArrayList<>(); - private List filteredContours_ = new ArrayList<>(); - private List deSpeckledContours_ = new ArrayList<>(); - private List groupedContours_ = new ArrayList<>(); - - private static final Scalar contourRectColor = new Scalar(255, 0, 0); - private static final Scalar BoxRectColor = new Scalar(0, 0, 233); - - private void drawContour(Mat inputMat, RotatedRect contourRect) { - if (contourRect == null) return; - List drawnContour = new ArrayList<>(); - Point[] vertices = new Point[4]; - contourRect.points(vertices); - MatOfPoint contour = new MatOfPoint(vertices); - drawnContour.add(contour); - Rect box = Imgproc.boundingRect(contour); - Imgproc.drawContours(inputMat, drawnContour, 0, contourRectColor, 3); - Imgproc.circle(inputMat, contourRect.center, 3, contourRectColor); - Imgproc.rectangle(inputMat, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), BoxRectColor, 2); - } - - PipelineResult runPipeline(Pipeline currentPipeline, Mat inputImage, Mat outputImage, CameraValues cameraValues, boolean shouldFlip, boolean driverMode) { - var pipelineResult = new PipelineResult(); - - if (currentPipeline == null) { - return pipelineResult; - } - - // flip the image - if (shouldFlip) { - Core.flip(inputImage, inputImage, -1); - } - - // if we're in driver mode don't do anything, and return a blank result - if (driverMode) { - inputImage.copyTo(outputImage); - return pipelineResult; - } - - foundContours_.clear(); - filteredContours_.clear(); - deSpeckledContours_.clear(); - groupedContours_.clear(); - - // HSV threshold the image - Scalar hsvLower = new Scalar(currentPipeline.hue.get(0).intValue(), currentPipeline.saturation.get(0).intValue(), currentPipeline.value.get(0).intValue()); - Scalar hsvUpper = new Scalar(currentPipeline.hue.get(1).intValue(), currentPipeline.saturation.get(1).intValue(), currentPipeline.value.get(1).intValue()); - hsvThreshold(inputImage, hsvThreshMat, hsvLower, hsvUpper, currentPipeline.erode, currentPipeline.dilate); - - // Make sure we're BFR - if (currentPipeline.isBinary) { - Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); - } else { - inputImage.copyTo(outputImage); - } - - // search for contours - foundContours_ = findContours(hsvThreshMat); - if (foundContours_.size() < 1) { - return pipelineResult; - } - - // filter contours by area, ratio and extent - filteredContours_ = filterContours(foundContours_, currentPipeline.area, currentPipeline.ratio, currentPipeline.extent); - if (filteredContours_.size() < 1) { - return pipelineResult; - } - - // reject "speckle" contours - deSpeckledContours_ = rejectSpeckles(filteredContours_, currentPipeline.speckle.doubleValue()); - if (deSpeckledContours_.size() < 1) { - return pipelineResult; - } - - // group targets - groupedContours_ = groupTargets(deSpeckledContours_, currentPipeline.targetIntersection, currentPipeline.targetGroup); - if (groupedContours_.size() < 1) { - return pipelineResult; - } - - // sort targets down to our final target - var finalRect = sortTargetsToOne(groupedContours_, currentPipeline.sortMode); - pipelineResult.RawPoint = finalRect; - pipelineResult.IsValid = true; - switch (currentPipeline.calibrationMode) { - case None: - ///use the center of the USBCamera to find the pitch and yaw difference - pipelineResult.CalibratedX = cameraValues.CenterX; - pipelineResult.CalibratedY = cameraValues.CenterY; - break; - case Single: - // use the static point as a calibration method instead of the center - pipelineResult.CalibratedX = currentPipeline.point.get(0).doubleValue(); - pipelineResult.CalibratedY = currentPipeline.point.get(1).doubleValue(); - break; - case Dual: - // use the calculated line to find the difference in length between the point and the line - pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; - pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; - break; - } - - pipelineResult.Pitch = cameraValues.CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); - pipelineResult.Yaw = cameraValues.CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); - pipelineResult.Area = finalRect.size.area(); - drawContour(outputImage, finalRect); - - return pipelineResult; - } - - void hsvThreshold(Mat srcImage, Mat dst, @NotNull Scalar hsvLower, @NotNull Scalar hsvUpper, boolean shouldErode, boolean shouldDilate) { - Imgproc.cvtColor(srcImage, hsvImage, Imgproc.COLOR_RGB2HSV, 3); - Imgproc.blur(hsvImage, hsvImage, blur); - Core.inRange(hsvImage, hsvLower, hsvUpper, dst); - if (shouldErode) { - Imgproc.erode(dst, dst, kernel); - } - if (shouldDilate) { - Imgproc.dilate(dst, dst, kernel); - } - hsvImage.release(); - } - - List findContours(Mat src) { - src.copyTo(binaryMat); - foundContours.clear(); - Imgproc.findContours(binaryMat, foundContours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); - binaryMat.release(); - return foundContours; - } - - List filterContours(List inputContours, List area, List ratio, List extent) { - for (MatOfPoint Contour : inputContours) { - try { - double contourArea = Imgproc.contourArea(Contour); - double AreaRatio = (contourArea / cameraValues.ImageArea) * 100; - double minArea = (MathHandler.sigmoid(area.get(0))); - double maxArea = (MathHandler.sigmoid(area.get(1))); - if (AreaRatio < minArea || AreaRatio > maxArea) { - continue; - } - var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); - - var targetFullness = contourArea; - double minExtent = (double) (extent.get(0).doubleValue() * rect.size.area()) / 100; - double maxExtent = (double) (extent.get(1).doubleValue() * rect.size.area()) / 100; - if (targetFullness <= minExtent || contourArea >= maxExtent) { - continue; - } - Rect bb = Imgproc.boundingRect(Contour); - double aspectRatio = (bb.width / bb.height); - if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { - continue; - } - filteredContours.add(Contour); - } catch (Exception e) { - System.err.println("Error while filtering contours"); - e.printStackTrace(); - } - } - return filteredContours; - } - - List rejectSpeckles(List inputContours, Double minimumPercentOfAverage) { - double averageArea = 0.0; - for (MatOfPoint c : inputContours) { - averageArea += Imgproc.contourArea(c); - } - averageArea /= inputContours.size(); - var minimumAllowableArea = minimumPercentOfAverage / 100.0 * averageArea; - speckleRejectedContours.clear(); - for (MatOfPoint c : inputContours) { - if (Imgproc.contourArea(c) >= minimumAllowableArea) speckleRejectedContours.add(c); - } - return speckleRejectedContours; - } - - - private double calcDistance(RotatedRect rect) { - return FastMath.sqrt(FastMath.pow(cameraValues.CenterX - rect.center.x, 2) + FastMath.pow(cameraValues.CenterY - rect.center.y, 2)); - } - - private double calcMomentsX(MatOfPoint c) { - Moments m = Imgproc.moments(c); - return (m.get_m10() / m.get_m00()); - } - - RotatedRect sortTargetsToOne(List inputRects, SortMode sortMode) { - switch (sortMode) { - case Largest: - return Collections.max(inputRects, Comparator.comparing(rect -> rect.size.area())); - case Smallest: - return Collections.min(inputRects, Comparator.comparing(rect -> rect.size.area())); - case Highest: - return Collections.min(inputRects, Comparator.comparing(rect -> rect.center.y)); - case Lowest: - return Collections.max(inputRects, Comparator.comparing(rect -> rect.center.y)); - case Leftmost: - return Collections.min(inputRects, Comparator.comparing(rect -> rect.center.x)); - case Rightmost: - return Collections.max(inputRects, Comparator.comparing(rect -> rect.center.x)); - case Centermost: - return Collections.min(inputRects, sortByCentermostComparator); - default: - return inputRects.get(0); // default to whatever the first contour is, but this should never happen - } - } - - List groupTargets(List inputContours, TargetIntersection intersectionPoint, TargetGroup targetGroup) { - finalCountours.clear(); - inputContours.sort(sortByMomentsX); - Collections.reverse(inputContours); - if (targetGroup.equals(TargetGroup.Dual)) { - for (var i = 0; i < inputContours.size(); i++) { - List FinalContourList = new ArrayList<>(inputContours.get(i).toList()); - try { - MatOfPoint firstContour = inputContours.get(i); - MatOfPoint secondContour = inputContours.get(i + 1); - if (isIntersecting(firstContour, secondContour, intersectionPoint)) { - FinalContourList.addAll(secondContour.toList()); - } else { - FinalContourList.clear(); - continue; - } - firstContour.release(); - secondContour.release(); - MatOfPoint2f contour = new MatOfPoint2f(); - contour.fromList(FinalContourList); - if (contour.cols() != 0 && contour.rows() != 0) { - RotatedRect rect = Imgproc.minAreaRect(contour); - finalCountours.add(rect); - } - } catch (IndexOutOfBoundsException e) { - FinalContourList.clear(); - } - } - - } else if (targetGroup.equals(TargetGroup.Single)) { - for (MatOfPoint inputContour : inputContours) { - MatOfPoint2f contour = new MatOfPoint2f(); - contour.fromArray(inputContour.toArray()); - if (contour.cols() != 0 && contour.rows() != 0) { - RotatedRect rect = Imgproc.minAreaRect(contour); - finalCountours.add(rect); - } - } - } - return finalCountours; - } - - private boolean isIntersecting(MatOfPoint ContourOne, MatOfPoint ContourTwo, TargetIntersection intersectionPoint) { - if (intersectionPoint.equals(TargetIntersection.None)) { - return true; - } - try { - intersectMatA.fromArray(ContourOne.toArray()); - intersectMatB.fromArray(ContourTwo.toArray()); - RotatedRect a = Imgproc.fitEllipse(intersectMatA); - RotatedRect b = Imgproc.fitEllipse(intersectMatB); - double mA = MathHandler.toSlope(a.angle); - double mB = MathHandler.toSlope(b.angle); - double x0A = a.center.x; - double y0A = a.center.y; - double x0B = b.center.x; - double y0B = b.center.y; - double intersectionX = ((mA * x0A) - y0A - (mB * x0B) + y0B) / (mA - mB); - double intersectionY = (mA * (intersectionX - x0A)) + y0A; - double massX = (x0A + x0B) / 2; - double massY = (y0A + y0B) / 2; - switch (intersectionPoint) { - case Up: { - if (intersectionY < massY) { - if (mA > 0 && mB < 0) { - return true; - } - } - break; - } - case Down: { - if (intersectionY > massY) { - if (mA < 0 && mB > 0) { - return true; - } - } - - break; - } - case Left: { - if (intersectionX < massX) { - - return true; - } - break; - } - case Right: { - if (intersectionX > massX) { - return true; - } - break; - } - } - return false; - } catch (Exception e) { - return false; - } - } +public interface CVProcess { + PipelineResult runPipeline(Pipeline currentPipeline, Mat inputImage, Mat outputImage, CameraValues cameraValues, boolean shouldFlip, boolean driverMode); } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java new file mode 100644 index 000000000..e7a0585b4 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java @@ -0,0 +1,345 @@ +package com.chameleonvision.vision.process; + +import com.chameleonvision.vision.*; +import com.chameleonvision.vision.camera.CameraValues; +import com.chameleonvision.util.MathHandler; +import org.apache.commons.math3.util.FastMath; +import org.jetbrains.annotations.NotNull; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; +import org.opencv.imgproc.Moments; + +import java.util.*; + +@SuppressWarnings("WeakerAccess") +public class StandardCVProcess implements CVProcess { + + private final CameraValues cameraValues; + private Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5, 5)); + private Size blur = new Size(3, 3); + private Mat hsvImage = new Mat(); + private List foundContours = new ArrayList<>(); + private Mat binaryMat = new Mat(); + private List filteredContours = new ArrayList<>(); + private Comparator sortByCentermostComparator = Comparator.comparingDouble(this::calcDistance); + private List speckleRejectedContours = new ArrayList<>(); + private Comparator sortByMomentsX = Comparator.comparingDouble(this::calcMomentsX); + private List finalCountours = new ArrayList<>(); + private MatOfPoint2f intersectMatA = new MatOfPoint2f(); + private MatOfPoint2f intersectMatB = new MatOfPoint2f(); + + StandardCVProcess(CameraValues cameraValues) { + this.cameraValues = cameraValues; + } + + private Mat cameraInputMat = new Mat(); + private Mat hsvThreshMat = new Mat(); + private Mat streamOutputMat = new Mat(); + + private List foundContours_ = new ArrayList<>(); + private List filteredContours_ = new ArrayList<>(); + private List deSpeckledContours_ = new ArrayList<>(); + private List groupedContours_ = new ArrayList<>(); + + private static final Scalar contourRectColor = new Scalar(255, 0, 0); + private static final Scalar BoxRectColor = new Scalar(0, 0, 233); + + private void drawContour(Mat inputMat, RotatedRect contourRect) { + if (contourRect == null) return; + List drawnContour = new ArrayList<>(); + Point[] vertices = new Point[4]; + contourRect.points(vertices); + MatOfPoint contour = new MatOfPoint(vertices); + drawnContour.add(contour); + Rect box = Imgproc.boundingRect(contour); + Imgproc.drawContours(inputMat, drawnContour, 0, contourRectColor, 3); + Imgproc.circle(inputMat, contourRect.center, 3, contourRectColor); + Imgproc.rectangle(inputMat, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), BoxRectColor, 2); + } + + public PipelineResult runPipeline(Pipeline currentPipeline, Mat inputImage, Mat outputImage, CameraValues cameraValues, boolean shouldFlip, boolean driverMode) { + var pipelineResult = new PipelineResult(); + + if (currentPipeline == null) { + return pipelineResult; + } + + // flip the image + if (shouldFlip) { + Core.flip(inputImage, inputImage, -1); + } + + // if we're in driver mode don't do anything, and return a blank result + if (driverMode) { + inputImage.copyTo(outputImage); + return pipelineResult; + } + + foundContours_.clear(); + filteredContours_.clear(); + deSpeckledContours_.clear(); + groupedContours_.clear(); + + // HSV threshold the image + Scalar hsvLower = new Scalar(currentPipeline.hue.get(0).intValue(), currentPipeline.saturation.get(0).intValue(), currentPipeline.value.get(0).intValue()); + Scalar hsvUpper = new Scalar(currentPipeline.hue.get(1).intValue(), currentPipeline.saturation.get(1).intValue(), currentPipeline.value.get(1).intValue()); + hsvThreshold(inputImage, hsvThreshMat, hsvLower, hsvUpper, currentPipeline.erode, currentPipeline.dilate); + + // Make sure we're BFR + if (currentPipeline.isBinary) { + Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); + } else { + inputImage.copyTo(outputImage); + } + + // search for contours + foundContours_ = findContours(hsvThreshMat); + if (foundContours_.size() < 1) { + return pipelineResult; + } + + // filter contours by area, ratio and extent + filteredContours_ = filterContours(foundContours_, currentPipeline.area, currentPipeline.ratio, currentPipeline.extent); + if (filteredContours_.size() < 1) { + return pipelineResult; + } + + // reject "speckle" contours + deSpeckledContours_ = rejectSpeckles(filteredContours_, currentPipeline.speckle.doubleValue()); + if (deSpeckledContours_.size() < 1) { + return pipelineResult; + } + + // group targets + groupedContours_ = groupTargets(deSpeckledContours_, currentPipeline.targetIntersection, currentPipeline.targetGroup); + if (groupedContours_.size() < 1) { + return pipelineResult; + } + + // sort targets down to our final target + var finalRect = sortTargetsToOne(groupedContours_, currentPipeline.sortMode); + pipelineResult.RawPoint = finalRect; + pipelineResult.IsValid = true; + switch (currentPipeline.calibrationMode) { + case None: + ///use the center of the USBCamera to find the pitch and yaw difference + pipelineResult.CalibratedX = cameraValues.CenterX; + pipelineResult.CalibratedY = cameraValues.CenterY; + break; + case Single: + // use the static point as a calibration method instead of the center + pipelineResult.CalibratedX = currentPipeline.point.get(0).doubleValue(); + pipelineResult.CalibratedY = currentPipeline.point.get(1).doubleValue(); + break; + case Dual: + // use the calculated line to find the difference in length between the point and the line + pipelineResult.CalibratedX = (finalRect.center.y - currentPipeline.b) / currentPipeline.m; + pipelineResult.CalibratedY = (finalRect.center.x * currentPipeline.m) + currentPipeline.b; + break; + } + + pipelineResult.Pitch = cameraValues.CalculatePitch(finalRect.center.y, pipelineResult.CalibratedY); + pipelineResult.Yaw = cameraValues.CalculateYaw(finalRect.center.x, pipelineResult.CalibratedX); + pipelineResult.Area = finalRect.size.area(); + drawContour(outputImage, finalRect); + + return pipelineResult; + } + + void hsvThreshold(Mat srcImage, Mat dst, @NotNull Scalar hsvLower, @NotNull Scalar hsvUpper, boolean shouldErode, boolean shouldDilate) { + Imgproc.cvtColor(srcImage, hsvImage, Imgproc.COLOR_RGB2HSV, 3); + Imgproc.blur(hsvImage, hsvImage, blur); + Core.inRange(hsvImage, hsvLower, hsvUpper, dst); + if (shouldErode) { + Imgproc.erode(dst, dst, kernel); + } + if (shouldDilate) { + Imgproc.dilate(dst, dst, kernel); + } + hsvImage.release(); + } + + List findContours(Mat src) { + src.copyTo(binaryMat); + foundContours.clear(); + Imgproc.findContours(binaryMat, foundContours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); + binaryMat.release(); + return foundContours; + } + + List filterContours(List inputContours, List area, List ratio, List extent) { + for (MatOfPoint Contour : inputContours) { + try { + double contourArea = Imgproc.contourArea(Contour); + double AreaRatio = (contourArea / cameraValues.ImageArea) * 100; + double minArea = (MathHandler.sigmoid(area.get(0))); + double maxArea = (MathHandler.sigmoid(area.get(1))); + if (AreaRatio < minArea || AreaRatio > maxArea) { + continue; + } + var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); + + var targetFullness = contourArea; + double minExtent = (double) (extent.get(0).doubleValue() * rect.size.area()) / 100; + double maxExtent = (double) (extent.get(1).doubleValue() * rect.size.area()) / 100; + if (targetFullness <= minExtent || contourArea >= maxExtent) { + continue; + } + Rect bb = Imgproc.boundingRect(Contour); + double aspectRatio = (bb.width / bb.height); + if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { + continue; + } + filteredContours.add(Contour); + } catch (Exception e) { + System.err.println("Error while filtering contours"); + e.printStackTrace(); + } + } + return filteredContours; + } + + List rejectSpeckles(List inputContours, Double minimumPercentOfAverage) { + double averageArea = 0.0; + for (MatOfPoint c : inputContours) { + averageArea += Imgproc.contourArea(c); + } + averageArea /= inputContours.size(); + var minimumAllowableArea = minimumPercentOfAverage / 100.0 * averageArea; + speckleRejectedContours.clear(); + for (MatOfPoint c : inputContours) { + if (Imgproc.contourArea(c) >= minimumAllowableArea) speckleRejectedContours.add(c); + } + return speckleRejectedContours; + } + + + private double calcDistance(RotatedRect rect) { + return FastMath.sqrt(FastMath.pow(cameraValues.CenterX - rect.center.x, 2) + FastMath.pow(cameraValues.CenterY - rect.center.y, 2)); + } + + private double calcMomentsX(MatOfPoint c) { + Moments m = Imgproc.moments(c); + return (m.get_m10() / m.get_m00()); + } + + RotatedRect sortTargetsToOne(List inputRects, SortMode sortMode) { + switch (sortMode) { + case Largest: + return Collections.max(inputRects, Comparator.comparing(rect -> rect.size.area())); + case Smallest: + return Collections.min(inputRects, Comparator.comparing(rect -> rect.size.area())); + case Highest: + return Collections.min(inputRects, Comparator.comparing(rect -> rect.center.y)); + case Lowest: + return Collections.max(inputRects, Comparator.comparing(rect -> rect.center.y)); + case Leftmost: + return Collections.min(inputRects, Comparator.comparing(rect -> rect.center.x)); + case Rightmost: + return Collections.max(inputRects, Comparator.comparing(rect -> rect.center.x)); + case Centermost: + return Collections.min(inputRects, sortByCentermostComparator); + default: + return inputRects.get(0); // default to whatever the first contour is, but this should never happen + } + } + + List groupTargets(List inputContours, TargetIntersection intersectionPoint, TargetGroup targetGroup) { + finalCountours.clear(); + inputContours.sort(sortByMomentsX); + Collections.reverse(inputContours); + if (targetGroup.equals(TargetGroup.Dual)) { + for (var i = 0; i < inputContours.size(); i++) { + List FinalContourList = new ArrayList<>(inputContours.get(i).toList()); + try { + MatOfPoint firstContour = inputContours.get(i); + MatOfPoint secondContour = inputContours.get(i + 1); + if (isIntersecting(firstContour, secondContour, intersectionPoint)) { + FinalContourList.addAll(secondContour.toList()); + } else { + FinalContourList.clear(); + continue; + } + firstContour.release(); + secondContour.release(); + MatOfPoint2f contour = new MatOfPoint2f(); + contour.fromList(FinalContourList); + if (contour.cols() != 0 && contour.rows() != 0) { + RotatedRect rect = Imgproc.minAreaRect(contour); + finalCountours.add(rect); + } + } catch (IndexOutOfBoundsException e) { + FinalContourList.clear(); + } + } + + } else if (targetGroup.equals(TargetGroup.Single)) { + for (MatOfPoint inputContour : inputContours) { + MatOfPoint2f contour = new MatOfPoint2f(); + contour.fromArray(inputContour.toArray()); + if (contour.cols() != 0 && contour.rows() != 0) { + RotatedRect rect = Imgproc.minAreaRect(contour); + finalCountours.add(rect); + } + } + } + return finalCountours; + } + + private boolean isIntersecting(MatOfPoint ContourOne, MatOfPoint ContourTwo, TargetIntersection intersectionPoint) { + if (intersectionPoint.equals(TargetIntersection.None)) { + return true; + } + try { + intersectMatA.fromArray(ContourOne.toArray()); + intersectMatB.fromArray(ContourTwo.toArray()); + RotatedRect a = Imgproc.fitEllipse(intersectMatA); + RotatedRect b = Imgproc.fitEllipse(intersectMatB); + double mA = MathHandler.toSlope(a.angle); + double mB = MathHandler.toSlope(b.angle); + double x0A = a.center.x; + double y0A = a.center.y; + double x0B = b.center.x; + double y0B = b.center.y; + double intersectionX = ((mA * x0A) - y0A - (mB * x0B) + y0B) / (mA - mB); + double intersectionY = (mA * (intersectionX - x0A)) + y0A; + double massX = (x0A + x0B) / 2; + double massY = (y0A + y0B) / 2; + switch (intersectionPoint) { + case Up: { + if (intersectionY < massY) { + if (mA > 0 && mB < 0) { + return true; + } + } + break; + } + case Down: { + if (intersectionY > massY) { + if (mA < 0 && mB > 0) { + return true; + } + } + + break; + } + case Left: { + if (intersectionX < massX) { + + return true; + } + break; + } + case Right: { + if (intersectionX > massX) { + return true; + } + break; + } + } + return false; + } catch (Exception e) { + return false; + } + } +} diff --git a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java index 4c5b234de..b8a4fa360 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java @@ -7,7 +7,6 @@ import com.chameleonvision.web.ServerHandler; import edu.wpi.cscore.VideoException; import edu.wpi.first.networktables.*; import org.opencv.core.*; -import org.opencv.imgproc.Imgproc; import java.util.ArrayList; import java.util.HashMap; @@ -31,10 +30,10 @@ public class VisionProcess implements Runnable { private Pipeline currentPipeline; private CVProcess cvProcess; // pipeline process items - private List foundContours = new ArrayList<>(); - private List filteredContours = new ArrayList<>(); - private List deSpeckledContours = new ArrayList<>(); - private List groupedContours = new ArrayList<>(); +// private List foundContours = new ArrayList<>(); +// private List filteredContours = new ArrayList<>(); +// private List deSpeckledContours = new ArrayList<>(); +// private List groupedContours = new ArrayList<>(); private Mat cameraInputMat = new Mat(); private Mat hsvThreshMat = new Mat(); private Mat streamOutputMat = new Mat(); @@ -43,7 +42,7 @@ public class VisionProcess implements Runnable { public VisionProcess(CameraProcess cameraProcess) { // USBCamera settings - cvProcess = new CVProcess(cameraProcess.getCamVals()); + cvProcess = new StandardCVProcess(cameraProcess.getCamVals()); this.cameraProcess = cameraProcess; // new USBCameraProcess(cameraProcess); this.cameraName = cameraProcess.getCamName(); @@ -124,10 +123,10 @@ public class VisionProcess implements Runnable { while (!Thread.interrupted()) { startTime = System.nanoTime(); if ((startTime - lastFrameEndNanosec) * 1e-6 >= 1000.0 / (maxFps + 3)) { // 3 additional fps to allow for overhead - foundContours.clear(); - filteredContours.clear(); - groupedContours.clear(); - deSpeckledContours.clear(); +// foundContours.clear(); +// filteredContours.clear(); +// groupedContours.clear(); +// deSpeckledContours.clear(); // update FPS for ui only every 0.5 seconds if ((startTime - fpsLastTime) * 1e-6 >= 500) { From 1bca800f4f3599d114436efce5f5ba57d63a5b3e Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Mon, 4 Nov 2019 02:23:13 -0500 Subject: [PATCH 06/84] Begin further work on abstraction layout in separate package --- Main/chameleon-vision.iml | 3 - .../camera/CameraProcess.java | 11 ++++ .../camera/CameraProperties.java | 57 +++++++++++++++++++ .../camera/CameraStaticProperties.java | 36 ++++++++++++ .../camera/CameraStreamer.java | 7 +++ .../classabstraction/camera/USBCamera.java | 15 +++++ .../classabstraction/pipeline/CVPipeline.java | 21 +++++++ .../pipeline/CVPipeline2dSettings.java | 4 ++ .../pipeline/CVPipeline3dSettings.java | 4 ++ .../pipeline/CVPipelineSettings.java | 30 ++++++++++ .../pipeline/DriverVisionPipeline.java | 25 ++++++++ .../vision/camera/CameraManager.java | 4 +- .../vision/camera/CameraValues.java | 41 +++++++------ 13 files changed, 232 insertions(+), 26 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 13d0e774e..bb72d8934 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -10,9 +10,6 @@ - - - diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java new file mode 100644 index 000000000..fe5777c0c --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java @@ -0,0 +1,11 @@ +package com.chameleonvision.classabstraction.camera; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; + +public interface CameraProcess { + Pair getFrame(); + + void setExposure(int exposure); + void setBrightness(int brightness); +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java new file mode 100644 index 000000000..57a78d2fb --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java @@ -0,0 +1,57 @@ +package com.chameleonvision.classabstraction.camera; + +import edu.wpi.cscore.UsbCamera; +import edu.wpi.cscore.VideoMode; +import org.apache.commons.math3.util.FastMath; + +import java.util.Arrays; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class CameraProperties { + private static final double DEFAULT_FOV = 70; + private static final int DEFAULT_EXPOSURE = 50; + private static final int DEFAULT_BRIGHTNESS = 50; + private static final int MINIMUM_FPS = 30; + private static final int MINIMUM_WIDTH = 320; + private static final int MINIMUM_HEIGHT = 200; + private static final int MAX_INIT_MS = 1500; + private static final List ALLOWED_PIXEL_FORMATS = Arrays.asList(VideoMode.PixelFormat.kYUYV, VideoMode.PixelFormat.kMJPEG); + + private static final Predicate kMinFPSPredicate = (videoMode -> videoMode.fps >= MINIMUM_FPS); + private static final Predicate kMinSizePredicate = (videoMode -> videoMode.width >= MINIMUM_FPS && videoMode.height >= MINIMUM_FPS); + private static final Predicate kPixelFormatPredicate = (videoMode -> ALLOWED_PIXEL_FORMATS.contains(videoMode.pixelFormat)); + + public CameraStaticProperties staticProperties; + public final double FOV; + public final List videoModes; + + public CameraProperties(UsbCamera baseCamera, double fov) { + FOV = fov; + + videoModes = filterVideoModes(baseCamera.enumerateVideoModes()); + + + } + + private List filterVideoModes(VideoMode[] videoModes) { + Predicate fullPredicate = kMinFPSPredicate.and(kMinSizePredicate).and(kPixelFormatPredicate); + Stream validModes = Arrays.stream(videoModes).filter(fullPredicate); + return validModes.collect(Collectors.toList()); + } + + public void updateVideoMode(VideoMode videoMode) { + staticProperties = new CameraStaticProperties(videoMode.width, videoMode.height, FOV); + } + + public double CalculatePitch(double PixelY, double centerY) { + double pitch = FastMath.toDegrees(FastMath.atan((PixelY - centerY) / staticProperties.VerticalFocalLength)); + return (pitch * -1); + } + + public double CalculateYaw(double PixelX, double centerX) { + return FastMath.toDegrees(FastMath.atan((PixelX - centerX) / staticProperties.HorizontalFocalLength)); + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java new file mode 100644 index 000000000..7242fb3c4 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java @@ -0,0 +1,36 @@ +package com.chameleonvision.classabstraction.camera; + +import org.apache.commons.math3.fraction.Fraction; +import org.apache.commons.math3.util.FastMath; + +public class CameraStaticProperties { + + public final int ImageWidth; + public final int ImageHeight; + public final double FOV; + public final double ImageArea; + public final double CenterX; + public final double CenterY; + public final double HorizontalFocalLength; + public final double VerticalFocalLength; + + public CameraStaticProperties(int imageWidth, int imageHeight, double fov) { + ImageWidth = imageWidth; + ImageHeight = imageHeight; + FOV = fov; + ImageArea = ImageWidth * ImageHeight; + CenterX = ((double) ImageWidth / 2) - 0.5; + CenterY = ((double) ImageHeight / 2) - 0.5; + + // pinhole model calculations + double diagonalView = FastMath.toRadians(FOV); + Fraction aspectFraction = new Fraction(ImageWidth, ImageHeight); + int horizontalRatio = aspectFraction.getNumerator(); + int verticalRatio = aspectFraction.getDenominator(); + double diagonalAspect = FastMath.hypot(horizontalRatio, verticalRatio); + double horizontalView = FastMath.atan(FastMath.tan(diagonalView / 2) * (horizontalRatio / diagonalAspect)) * 2; + double verticalView = FastMath.atan(FastMath.tan(diagonalView / 2) * (verticalRatio / diagonalAspect)) * 2; + HorizontalFocalLength = ImageWidth / (2 * FastMath.tan(horizontalView /2)); + VerticalFocalLength = ImageHeight / (2 * FastMath.tan(verticalView /2)); + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java new file mode 100644 index 000000000..fc3dc2a16 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java @@ -0,0 +1,7 @@ +package com.chameleonvision.classabstraction.camera; + +import org.opencv.core.Mat; + +public interface CameraStreamer { + void streamFrame(Mat frame); +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java new file mode 100644 index 000000000..e2212f8f2 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java @@ -0,0 +1,15 @@ +package com.chameleonvision.classabstraction.camera; + +import edu.wpi.cscore.UsbCamera; +import edu.wpi.cscore.VideoMode; + +public class USBCamera { + private final UsbCamera baseCamera; + private final CameraProperties properties; + + public USBCamera(UsbCamera camera) { + baseCamera = camera; + VideoMode vidMode = new VideoMode(VideoMode.PixelFormat.kYUYV, 640, 480, 60); + properties = new CameraProperties(baseCamera, 75); + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java new file mode 100644 index 000000000..fc057b790 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java @@ -0,0 +1,21 @@ +package com.chameleonvision.classabstraction.pipeline; + +import org.opencv.core.Mat; + +/** + * + * @param Pipeline result type + */ +public abstract class CVPipeline { + private CVPipelineSettings settings; + private Mat inputMat; + protected Mat outputMat; + + public CVPipeline(CVPipelineSettings settings) { + this.settings = settings; + } + + abstract void initPipeline(); + abstract R runPipeline(Mat inputMat); + abstract Mat getOutputMat(); +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java new file mode 100644 index 000000000..20dd868ba --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java @@ -0,0 +1,4 @@ +package com.chameleonvision.classabstraction.pipeline; + +public class CVPipeline2dSettings extends CVPipelineSettings { +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java new file mode 100644 index 000000000..b18a2e048 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java @@ -0,0 +1,4 @@ +package com.chameleonvision.classabstraction.pipeline; + +public abstract class CVPipeline3dSettings extends CVPipelineSettings { +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java new file mode 100644 index 000000000..c5f3281ae --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java @@ -0,0 +1,30 @@ +package com.chameleonvision.classabstraction.pipeline; + +import com.chameleonvision.vision.CalibrationMode; +import com.chameleonvision.vision.SortMode; +import com.chameleonvision.vision.TargetGroup; +import com.chameleonvision.vision.TargetIntersection; + +import java.util.Arrays; +import java.util.List; + +public abstract class CVPipelineSettings { + List hue = Arrays.asList(50, 180); + List saturation = Arrays.asList(50, 255); + List value = Arrays.asList(50, 255); + boolean erode = false; + boolean dilate = false; + List area = Arrays.asList(0.0, 100.0); + List ratio = Arrays.asList(0.0, 20.0); + List extent = Arrays.asList(0, 100); + Number speckle = 5; + boolean isBinary = false; + SortMode sortMode = SortMode.Largest; + TargetGroup targetGroup = TargetGroup.Single; + TargetIntersection targetIntersection = TargetIntersection.Up; + double m = 1; + double b = 0; + List point = Arrays.asList(0,0); + CalibrationMode calibrationMode = CalibrationMode.None; + String nickname = ""; +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java new file mode 100644 index 000000000..200c38eb9 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java @@ -0,0 +1,25 @@ +package com.chameleonvision.classabstraction.pipeline; + +import org.opencv.core.Mat; + +public class DriverVisionPipeline extends CVPipeline { + public DriverVisionPipeline(CVPipelineSettings settings) { + super(settings); + } + + @Override + void initPipeline() { + // set exposure/brightness of camera? + } + + @Override + Void runPipeline(Mat inputMat) { + this.outputMat = inputMat; + return null; + } + + @Override + Mat getOutputMat() { + return this.outputMat; + } +} diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java index a4855feec..d43f82ee4 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraManager.java @@ -43,7 +43,7 @@ public class CameraManager { public static HashMap getAllCamerasByName() { return allCamerasByName; } - public static List getAllCameraByNickname(){ + public static List getAllCameraByNickname() { var cameras = getAllCamerasByName(); return cameras.values().stream().map(USBCamera::getNickname).collect(Collectors.toList()); } @@ -101,6 +101,7 @@ public class CameraManager { if (curCam == null) throw new CameraException(CameraException.CameraExceptionType.BAD_CAMERA); return curCam; } + public static Integer getCurrentCameraIndex() throws CameraException { if (allCamerasByName.size() == 0) throw new CameraException(CameraException.CameraExceptionType.NO_CAMERA); List arr = new ArrayList<>(allCamerasByName.keySet()); @@ -118,6 +119,7 @@ public class CameraManager { SettingsManager.generalSettings.currentCamera = cameraName; SettingsManager.updateCameraSetting(cameraName, getCurrentCamera().getCurrentPipelineIndex()); } + public static void setCurrentCamera(int cameraIndex) throws CameraException { List s = new ArrayList(allCamerasByName.keySet()); setCurrentCamera(s.get(cameraIndex)); diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java index 50dcde294..8335f3bd6 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraValues.java @@ -3,7 +3,6 @@ package com.chameleonvision.vision.camera; import org.apache.commons.math3.fraction.Fraction; import org.apache.commons.math3.util.FastMath; -@SuppressWarnings("WeakerAccess") public class CameraValues { public final int ImageWidth; public final int ImageHeight; @@ -11,20 +10,14 @@ public class CameraValues { public final double ImageArea; public final double CenterX; public final double CenterY; - public final double DiagonalView; - public final double DiagonalAspect; - public final Fraction AspectFraction; - public final int HorizontalRatio; - public final int VerticalRatio; - public final double HorizontalView; - public final double VerticalView; - public final double HorizontalFocalLength; - public final double VerticalFocalLength; + private final double HorizontalFocalLength; + private final double VerticalFocalLength; public CameraValues(USBCamera USBCamera) { this(USBCamera.getVideoMode().width, USBCamera.getVideoMode().height, USBCamera.getFOV()); } + public CameraValues(int imageWidth, int imageHeight, double fov) { ImageWidth = imageWidth; ImageHeight = imageHeight; @@ -32,21 +25,25 @@ public class CameraValues { ImageArea = ImageWidth * ImageHeight; CenterX = ((double) ImageWidth / 2) - 0.5; CenterY = ((double) ImageHeight / 2) - 0.5; - DiagonalView = FastMath.toRadians(FOV); - AspectFraction = new Fraction(ImageWidth, ImageHeight); - HorizontalRatio = AspectFraction.getNumerator(); - VerticalRatio = AspectFraction.getDenominator(); - DiagonalAspect = FastMath.hypot(HorizontalRatio, VerticalRatio); - HorizontalView = FastMath.atan(FastMath.tan(DiagonalView / 2) * (HorizontalRatio / DiagonalAspect)) * 2; - VerticalView = FastMath.atan(FastMath.tan(DiagonalView / 2) * (VerticalRatio / DiagonalAspect)) * 2; - HorizontalFocalLength = ImageWidth / (2 * FastMath.tan(HorizontalView /2)); - VerticalFocalLength = ImageHeight / (2 * FastMath.tan(VerticalView /2)); + + // pinhole model calculations + double diagonalView = FastMath.toRadians(FOV); + Fraction aspectFraction = new Fraction(ImageWidth, ImageHeight); + int horizontalRatio = aspectFraction.getNumerator(); + int verticalRatio = aspectFraction.getDenominator(); + double diagonalAspect = FastMath.hypot(horizontalRatio, verticalRatio); + double horizontalView = FastMath.atan(FastMath.tan(diagonalView / 2) * (horizontalRatio / diagonalAspect)) * 2; + double verticalView = FastMath.atan(FastMath.tan(diagonalView / 2) * (verticalRatio / diagonalAspect)) * 2; + HorizontalFocalLength = ImageWidth / (2 * FastMath.tan(horizontalView /2)); + VerticalFocalLength = ImageHeight / (2 * FastMath.tan(verticalView /2)); } - public double CalculatePitch(double PixelY, double centerY){ - double pitch = FastMath.toDegrees(FastMath.atan((PixelY - centerY) / VerticalFocalLength)); + + public double CalculatePitch(double PixelY, double centerY) { + double pitch = FastMath.toDegrees(FastMath.atan((PixelY - centerY) / VerticalFocalLength)); return (pitch * -1); } - public double CalculateYaw(double PixelX, double centerX){ + + public double CalculateYaw(double PixelX, double centerX) { return FastMath.toDegrees(FastMath.atan((PixelX - centerX) / HorizontalFocalLength)); } } From d619d85109d1588cd1ae4249a8d26afe03d057b3 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 5 Nov 2019 07:47:15 -0800 Subject: [PATCH 07/84] Start filling in CVPipeline2d --- .../classabstraction/pipeline/CVPipeline.java | 2 +- .../pipeline/CVPipeline2d.java | 219 ++++++++++++++++++ .../pipeline/CVPipeline2dSettings.java | 4 - .../pipeline/CVPipeline3d.java | 33 +++ .../pipeline/CVPipeline3dSettings.java | 4 - .../pipeline/CVPipelineSettings.java | 12 +- .../pipeline/DriverVisionPipeline.java | 2 +- .../chameleonvision/vision/ImageFlipMode.java | 5 + 8 files changed, 265 insertions(+), 16 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java delete mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java delete mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java create mode 100644 Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java index fc057b790..ac6f93781 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java @@ -7,7 +7,7 @@ import org.opencv.core.Mat; * @param Pipeline result type */ public abstract class CVPipeline { - private CVPipelineSettings settings; + protected CVPipelineSettings settings; private Mat inputMat; protected Mat outputMat; diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java new file mode 100644 index 000000000..ffe35dae6 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java @@ -0,0 +1,219 @@ +package com.chameleonvision.classabstraction.pipeline; + +import com.chameleonvision.util.MathHandler; +import com.chameleonvision.vision.ImageFlipMode; +import com.chameleonvision.vision.camera.CameraValues; +import org.jetbrains.annotations.NotNull; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; + +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("WeakerAccess") +public class CVPipeline2d extends CVPipeline { + + private List foundContours_ = new ArrayList<>(); + private List filteredContours_ = new ArrayList<>(); + private List deSpeckledContours_ = new ArrayList<>(); + private List groupedContours_ = new ArrayList<>(); + + public CVPipeline2d(CVPipelineSettings settings) { + super(settings); + } + + @Override + void initPipeline() { + + } + + @Override + CVPipeline2d.CVPipeline2dResult runPipeline(Mat inputMat) { + var shouldFlip = settings.flipMode.equals(ImageFlipMode.BOTH); + var result = new CVPipeline2dResult(); + + // flip the image + if (shouldFlip) { + Core.flip(inputMat, inputMat, -1); + } + + foundContours_.clear(); + filteredContours_.clear(); + deSpeckledContours_.clear(); + groupedContours_.clear(); + + // HSV threshold the image + Scalar hsvLower = new Scalar(settings.hue.get(0).intValue(), settings.saturation.get(0).intValue(), settings.value.get(0).intValue()); + Scalar hsvUpper = new Scalar(settings.hue.get(1).intValue(), settings.saturation.get(1).intValue(), settings.value.get(1).intValue()); + hsvThreshold(inputImage, hsvThreshMat, settings.erode, settings.dilate); + + // Make sure we're BFR + if (settings.isBinary) { + Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); + } else { + inputImage.copyTo(outputImage); + } + + // search for contours + foundContours_ = findContours(hsvThreshMat); + if (foundContours_.size() < 1) { + return result; + } + + // filter contours by area, ratio and extent + filteredContours_ = filterContours(foundContours_, settings.area, settings.ratio, settings.extent); + if (filteredContours_.size() < 1) { + return result; + } + + // reject "speckle" contours + deSpeckledContours_ = rejectSpeckles(filteredContours_, settings.speckle.doubleValue()); + if (deSpeckledContours_.size() < 1) { + return result; + } + + // group targets + groupedContours_ = groupTargets(deSpeckledContours_, settings.targetIntersection, settings.targetGroup); + if (groupedContours_.size() < 1) { + return result; + } + + // sort targets down to our final target + var finalRect = sortTargetsToOne(groupedContours_, settings.sortMode); + result.RawPoint = finalRect; + result.IsValid = true; + switch (settings.calibrationMode) { + case None: + ///use the center of the USBCamera to find the pitch and yaw difference + result.CalibratedX = cameraValues.CenterX; + result.CalibratedY = cameraValues.CenterY; + break; + case Single: + // use the static point as a calibration method instead of the center + result.CalibratedX = settings.point.get(0).doubleValue(); + result.CalibratedY = settings.point.get(1).doubleValue(); + break; + case Dual: + // use the calculated line to find the difference in length between the point and the line + result.CalibratedX = (finalRect.center.y - settings.b) / settings.m; + result.CalibratedY = (finalRect.center.x * settings.m) + settings.b; + break; + } + + result.Pitch = cameraValues.CalculatePitch(finalRect.center.y, result.CalibratedY); + result.Yaw = cameraValues.CalculateYaw(finalRect.center.x, result.CalibratedX); + result.Area = finalRect.size.area(); + drawContour(outputImage, finalRect); + + return result; + } + + /** + * HSV Threshold a given image. Copies the HSV Thresholded image to the [dst] matrix with the given + * hsv settings and blur settings. Can also erode and dilate the image + * @param srcImage the source image, which is not mutated + * @param dst the destination image, which is mutated to save the result + * @param hsvLower the lower bound for the HSV settings + * @param hsvUpper the upper bound for the HSV settings + * @param kernel the kernal used to erode/dilate the image + * @param blur the size of the blur image + * @param shouldErode if we should erode + * @param shouldDilate if we should dilate + */ + public static void hsvThreshold(Mat srcImage, Mat dst, @NotNull Scalar hsvLower, + @NotNull Scalar hsvUpper, Mat kernel, Size blur, + boolean shouldErode, boolean shouldDilate) { + Imgproc.cvtColor(srcImage, dst, Imgproc.COLOR_RGB2HSV, 3); + Imgproc.blur(dst, dst, blur); + Core.inRange(dst, hsvLower, hsvUpper, dst); + if (shouldErode) { + Imgproc.erode(dst, dst, kernel); + } + if (shouldDilate) { + Imgproc.dilate(dst, dst, kernel); + } + dst.release(); + } + + /** + * Find contours from an image + * @param src the image we're looking at + * @param binaryMat a temporary image + * @param hierarchy the hierarchy of the image (just a new Mat();) + * @param emptyList a list to fill with stuff. Will be cleared + * @return the empty list, now full of contours + */ + public static List findContours(Mat src, Mat binaryMat, Mat hierarchy, List emptyList) { + src.copyTo(binaryMat); + emptyList.clear(); + Imgproc.findContours(binaryMat, emptyList, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); + binaryMat.release(); + return emptyList; + } + + public static List filterContours(List inputContours, List area, List ratio, List extent, CameraValues cameraValues) { + for (MatOfPoint Contour : inputContours) { + try { + double contourArea = Imgproc.contourArea(Contour); + double AreaRatio = (contourArea / cameraValues.ImageArea) * 100; + double minArea = (MathHandler.sigmoid(area.get(0))); + double maxArea = (MathHandler.sigmoid(area.get(1))); + if (AreaRatio < minArea || AreaRatio > maxArea) { + continue; + } + var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); + + var targetFullness = contourArea; + double minExtent = (double) (extent.get(0).doubleValue() * rect.size.area()) / 100; + double maxExtent = (double) (extent.get(1).doubleValue() * rect.size.area()) / 100; + if (targetFullness <= minExtent || contourArea >= maxExtent) { + continue; + } + Rect bb = Imgproc.boundingRect(Contour); + double aspectRatio = (bb.width / bb.height); + if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { + continue; + } + filteredContours.add(Contour); + } catch (Exception e) { + System.err.println("Error while filtering contours"); + e.printStackTrace(); + } + } + return filteredContours; + } + + @Override + Mat getOutputMat() { + return null; + } + + public static class CVPipeline2dSettings extends CVPipelineSettings { + double dualTargetCalibrationM = 1; + double dualTargetCalibrationB = 0; + } + + public static class CVPipeline2dResult { + public boolean hasTarget = false; + public ArrayList targets = new ArrayList<>(); // targets sorted by likelihood + + public CVPipeline2dResult(ArrayList targets, boolean hasTarget) { + this.targets = targets; + this.hasTarget = hasTarget; + } + + public CVPipeline2dResult() { + } + } + + public static class Target { + public boolean isValid = false; + public double calibratedX = 0.0; + public double calibratedY = 0.0; + public double pitch = 0.0; + public double yaw = 0.0; + public double area = 0.0; + RotatedRect rawPoint; + } + +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java deleted file mode 100644 index 20dd868ba..000000000 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.chameleonvision.classabstraction.pipeline; - -public class CVPipeline2dSettings extends CVPipelineSettings { -} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java new file mode 100644 index 000000000..7fce5f885 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java @@ -0,0 +1,33 @@ +package com.chameleonvision.classabstraction.pipeline; + +import org.opencv.core.Mat; + +public class CVPipeline3d extends CVPipeline { + + public CVPipeline3d(CVPipelineSettings settings) { + super(settings); + } + + @Override + void initPipeline() { + + } + + @Override + CVPipeline3d.CVPipeline3dResult runPipeline(Mat inputMat) { + return null; + } + + @Override + Mat getOutputMat() { + return null; + } + + public static class CVPipeline3dSettings extends CVPipelineSettings { + } + + public static class CVPipeline3dResult { + + } + +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java deleted file mode 100644 index b18a2e048..000000000 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.chameleonvision.classabstraction.pipeline; - -public abstract class CVPipeline3dSettings extends CVPipelineSettings { -} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java index c5f3281ae..79909b0fa 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java @@ -1,14 +1,13 @@ package com.chameleonvision.classabstraction.pipeline; -import com.chameleonvision.vision.CalibrationMode; -import com.chameleonvision.vision.SortMode; -import com.chameleonvision.vision.TargetGroup; -import com.chameleonvision.vision.TargetIntersection; +import com.chameleonvision.vision.*; import java.util.Arrays; import java.util.List; +@SuppressWarnings("ALL") public abstract class CVPipelineSettings { + ImageFlipMode flipMode = ImageFlipMode.NONE; List hue = Arrays.asList(50, 180); List saturation = Arrays.asList(50, 255); List value = Arrays.asList(50, 255); @@ -22,9 +21,10 @@ public abstract class CVPipelineSettings { SortMode sortMode = SortMode.Largest; TargetGroup targetGroup = TargetGroup.Single; TargetIntersection targetIntersection = TargetIntersection.Up; - double m = 1; - double b = 0; List point = Arrays.asList(0,0); CalibrationMode calibrationMode = CalibrationMode.None; + String nickname = ""; + double exposure = 50.0; + double brightness = 50.0; } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java index 200c38eb9..a0d985f40 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java @@ -9,7 +9,7 @@ public class DriverVisionPipeline extends CVPipeline { @Override void initPipeline() { - // set exposure/brightness of camera? + // TODO set exposure/brightness of camera } @Override diff --git a/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java b/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java new file mode 100644 index 000000000..5a778bd17 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java @@ -0,0 +1,5 @@ +package com.chameleonvision.vision; + +public enum ImageFlipMode { + NONE, VERTICAL, HORIZONTAL, BOTH +} From c213d4c75165e443d3704e2c94a119eea3e29a10 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Thu, 7 Nov 2019 11:15:54 -0500 Subject: [PATCH 08/84] Recreated all pipeline steps as Pipes and set up CVPipeline2d --- Main/chameleon-vision.iml | 4 +- .../camera/CameraProperties.java | 3 +- .../camera/CameraStaticProperties.java | 8 +- .../classabstraction/camera/USBCamera.java | 2 +- .../classabstraction/pipeline/CVPipeline.java | 10 +- .../pipeline/CVPipeline2d.java | 256 +++++++----------- .../pipeline/CVPipeline3d.java | 66 ++--- .../pipeline/CVPipelineResult.java | 11 + .../pipeline/DriverVisionPipeline.java | 22 +- .../pipeline/pipes/BlurPipe.java | 31 +++ .../pipeline/pipes/Collect2dTargetsPipe.java | 71 +++++ .../pipeline/pipes/Draw2dContoursPipe.java | 85 ++++++ .../pipeline/pipes/ErodeDilatePipe.java | 38 +++ .../pipeline/pipes/FilterContoursPipe.java | 62 +++++ .../pipeline/pipes/FindContoursPipe.java | 26 ++ .../pipeline/pipes/GroupContoursPipe.java | 154 +++++++++++ .../pipeline/pipes/HsvPipe.java | 35 +++ .../pipeline/pipes/OutputMatPipe.java | 31 +++ .../classabstraction/pipeline/pipes/Pipe.java | 13 + .../pipeline/pipes/RotateFlipPipe.java | 33 +++ .../pipeline/pipes/SortContoursPipe.java | 74 +++++ .../pipeline/pipes/SpeckleRejectPipe.java | 44 +++ .../chameleonvision/vision/ImageFlipMode.java | 11 +- .../chameleonvision/vision/ImageRotation.java | 16 ++ .../chameleonvision/vision/TargetGroup.java | 3 +- 25 files changed, 888 insertions(+), 221 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/BlurPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Draw2dContoursPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/ErodeDilatePipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FilterContoursPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FindContoursPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/GroupContoursPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/HsvPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/OutputMatPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Pipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/RotateFlipPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SortContoursPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SpeckleRejectPipe.java create mode 100644 Main/src/main/java/com/chameleonvision/vision/ImageRotation.java diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index bb72d8934..1799ea932 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -1,11 +1,13 @@ - + + + diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java index 57a78d2fb..4e0ab8a4d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java @@ -31,9 +31,8 @@ public class CameraProperties { public CameraProperties(UsbCamera baseCamera, double fov) { FOV = fov; + // TODO: determine how to set the initial videomode properly videoModes = filterVideoModes(baseCamera.enumerateVideoModes()); - - } private List filterVideoModes(VideoMode[] videoModes) { diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java index 7242fb3c4..5f9e9faee 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java @@ -9,8 +9,8 @@ public class CameraStaticProperties { public final int ImageHeight; public final double FOV; public final double ImageArea; - public final double CenterX; - public final double CenterY; + public final double centerX; + public final double centerY; public final double HorizontalFocalLength; public final double VerticalFocalLength; @@ -19,8 +19,8 @@ public class CameraStaticProperties { ImageHeight = imageHeight; FOV = fov; ImageArea = ImageWidth * ImageHeight; - CenterX = ((double) ImageWidth / 2) - 0.5; - CenterY = ((double) ImageHeight / 2) - 0.5; + centerX = ((double) ImageWidth / 2) - 0.5; + centerY = ((double) ImageHeight / 2) - 0.5; // pinhole model calculations double diagonalView = FastMath.toRadians(FOV); diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java index e2212f8f2..7175ff7f9 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java @@ -5,7 +5,7 @@ import edu.wpi.cscore.VideoMode; public class USBCamera { private final UsbCamera baseCamera; - private final CameraProperties properties; + public final CameraProperties properties; public USBCamera(UsbCamera camera) { baseCamera = camera; diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java index ac6f93781..b0a77c234 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java @@ -1,21 +1,21 @@ package com.chameleonvision.classabstraction.pipeline; +import com.chameleonvision.classabstraction.camera.USBCamera; import org.opencv.core.Mat; /** * * @param Pipeline result type */ -public abstract class CVPipeline { - protected CVPipelineSettings settings; +public abstract class CVPipeline { + protected S settings; private Mat inputMat; protected Mat outputMat; - public CVPipeline(CVPipelineSettings settings) { + public CVPipeline(S settings) { this.settings = settings; } - abstract void initPipeline(); + abstract void initPipeline(USBCamera camera); abstract R runPipeline(Mat inputMat); - abstract Mat getOutputMat(); } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java index ffe35dae6..213f96d10 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java @@ -1,191 +1,126 @@ package com.chameleonvision.classabstraction.pipeline; -import com.chameleonvision.util.MathHandler; -import com.chameleonvision.vision.ImageFlipMode; -import com.chameleonvision.vision.camera.CameraValues; -import org.jetbrains.annotations.NotNull; +import com.chameleonvision.classabstraction.camera.CameraStaticProperties; +import com.chameleonvision.classabstraction.camera.USBCamera; +import com.chameleonvision.classabstraction.pipeline.pipes.*; +import com.chameleonvision.vision.ImageRotation; +import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.*; import org.opencv.imgproc.Imgproc; -import java.util.ArrayList; import java.util.List; @SuppressWarnings("WeakerAccess") -public class CVPipeline2d extends CVPipeline { +public class CVPipeline2d extends CVPipeline { - private List foundContours_ = new ArrayList<>(); - private List filteredContours_ = new ArrayList<>(); - private List deSpeckledContours_ = new ArrayList<>(); - private List groupedContours_ = new ArrayList<>(); + private USBCamera camera; - public CVPipeline2d(CVPipelineSettings settings) { + private Mat rawCameraMat = new Mat(); + private Mat hsvOutputMat = new Mat(); + + public CVPipeline2d(CVPipeline2dSettings settings) { super(settings); } @Override - void initPipeline() { - + void initPipeline(USBCamera cam) { + camera = cam; } @Override CVPipeline2d.CVPipeline2dResult runPipeline(Mat inputMat) { - var shouldFlip = settings.flipMode.equals(ImageFlipMode.BOTH); - var result = new CVPipeline2dResult(); + long totalProcessTimeNanos = 0; + StringBuilder procTimeStringBuilder = new StringBuilder(); - // flip the image - if (shouldFlip) { - Core.flip(inputMat, inputMat, -1); - } + CameraStaticProperties camProps = camera.properties.staticProperties; - foundContours_.clear(); - filteredContours_.clear(); - deSpeckledContours_.clear(); - groupedContours_.clear(); + inputMat.copyTo(rawCameraMat); + + // prepare pipes + RotateFlipPipe rotateFlipPipe = new RotateFlipPipe(ImageRotation.DEG_0, settings.flipMode); + BlurPipe blurPipe = new BlurPipe(5); + ErodeDilatePipe erodeDilatePipe = new ErodeDilatePipe(settings.erode, settings.dilate, 7); - // HSV threshold the image Scalar hsvLower = new Scalar(settings.hue.get(0).intValue(), settings.saturation.get(0).intValue(), settings.value.get(0).intValue()); Scalar hsvUpper = new Scalar(settings.hue.get(1).intValue(), settings.saturation.get(1).intValue(), settings.value.get(1).intValue()); - hsvThreshold(inputImage, hsvThreshMat, settings.erode, settings.dilate); - // Make sure we're BFR - if (settings.isBinary) { - Imgproc.cvtColor(hsvThreshMat, outputImage, Imgproc.COLOR_GRAY2BGR, 3); - } else { - inputImage.copyTo(outputImage); - } + HsvPipe hsvPipe = new HsvPipe(hsvLower, hsvUpper); - // search for contours - foundContours_ = findContours(hsvThreshMat); - if (foundContours_.size() < 1) { - return result; - } + FindContoursPipe findContoursPipe = new FindContoursPipe(); + FilterContoursPipe filterContoursPipe = new FilterContoursPipe(settings.area, settings.ratio, settings.extent, camProps); + SpeckleRejectPipe speckleRejectPipe = new SpeckleRejectPipe(settings.speckle.doubleValue()); + GroupContoursPipe groupContoursPipe = new GroupContoursPipe(settings.targetGroup, settings.targetIntersection); + SortContoursPipe sortContoursPipe = new SortContoursPipe(settings.sortMode, camProps); + Collect2dTargetsPipe collect2dTargetsPipe = new Collect2dTargetsPipe(settings.calibrationMode, settings.point, + settings.dualTargetCalibrationM, settings.dualTargetCalibrationB, camProps); - // filter contours by area, ratio and extent - filteredContours_ = filterContours(foundContours_, settings.area, settings.ratio, settings.extent); - if (filteredContours_.size() < 1) { - return result; - } + OutputMatPipe outputMatPipe = new OutputMatPipe(settings.isBinary); - // reject "speckle" contours - deSpeckledContours_ = rejectSpeckles(filteredContours_, settings.speckle.doubleValue()); - if (deSpeckledContours_.size() < 1) { - return result; - } + Draw2dContoursPipe.Draw2dContoursSettings draw2dContoursSettings = new Draw2dContoursPipe.Draw2dContoursSettings(); + draw2dContoursSettings.showCentroid = false; + draw2dContoursSettings.showCrosshair = true; + draw2dContoursSettings.boxOutlineSize = 2; + draw2dContoursSettings.showRotatedBox = true; + draw2dContoursSettings.showMaximumBox = true; - // group targets - groupedContours_ = groupTargets(deSpeckledContours_, settings.targetIntersection, settings.targetGroup); - if (groupedContours_.size() < 1) { - return result; - } + Draw2dContoursPipe draw2dContoursPipe = new Draw2dContoursPipe(draw2dContoursSettings, camProps); - // sort targets down to our final target - var finalRect = sortTargetsToOne(groupedContours_, settings.sortMode); - result.RawPoint = finalRect; - result.IsValid = true; - switch (settings.calibrationMode) { - case None: - ///use the center of the USBCamera to find the pitch and yaw difference - result.CalibratedX = cameraValues.CenterX; - result.CalibratedY = cameraValues.CenterY; - break; - case Single: - // use the static point as a calibration method instead of the center - result.CalibratedX = settings.point.get(0).doubleValue(); - result.CalibratedY = settings.point.get(1).doubleValue(); - break; - case Dual: - // use the calculated line to find the difference in length between the point and the line - result.CalibratedX = (finalRect.center.y - settings.b) / settings.m; - result.CalibratedY = (finalRect.center.x * settings.m) + settings.b; - break; - } + // run pipes + Pair rotateFlipResult = rotateFlipPipe.run(inputMat); + totalProcessTimeNanos += rotateFlipResult.getRight(); + procTimeStringBuilder.append(String.format("RotateFlip: %.2fms, ", rotateFlipResult.getRight() / 1000.0)); - result.Pitch = cameraValues.CalculatePitch(finalRect.center.y, result.CalibratedY); - result.Yaw = cameraValues.CalculateYaw(finalRect.center.x, result.CalibratedX); - result.Area = finalRect.size.area(); - drawContour(outputImage, finalRect); + Pair blurResult = blurPipe.run(rotateFlipResult.getLeft()); + totalProcessTimeNanos += blurResult.getRight(); + procTimeStringBuilder.append(String.format("Blur: %.2fms, ", blurResult.getRight() / 1000.0)); - return result; - } + Pair erodeDilateResult = erodeDilatePipe.run(blurResult.getLeft()); + totalProcessTimeNanos += erodeDilateResult.getRight(); + procTimeStringBuilder.append(String.format("ErodeDilate: %.2fms, ", erodeDilateResult.getRight() / 1000.0)); - /** - * HSV Threshold a given image. Copies the HSV Thresholded image to the [dst] matrix with the given - * hsv settings and blur settings. Can also erode and dilate the image - * @param srcImage the source image, which is not mutated - * @param dst the destination image, which is mutated to save the result - * @param hsvLower the lower bound for the HSV settings - * @param hsvUpper the upper bound for the HSV settings - * @param kernel the kernal used to erode/dilate the image - * @param blur the size of the blur image - * @param shouldErode if we should erode - * @param shouldDilate if we should dilate - */ - public static void hsvThreshold(Mat srcImage, Mat dst, @NotNull Scalar hsvLower, - @NotNull Scalar hsvUpper, Mat kernel, Size blur, - boolean shouldErode, boolean shouldDilate) { - Imgproc.cvtColor(srcImage, dst, Imgproc.COLOR_RGB2HSV, 3); - Imgproc.blur(dst, dst, blur); - Core.inRange(dst, hsvLower, hsvUpper, dst); - if (shouldErode) { - Imgproc.erode(dst, dst, kernel); - } - if (shouldDilate) { - Imgproc.dilate(dst, dst, kernel); - } - dst.release(); - } + Pair hsvResult = hsvPipe.run(erodeDilateResult.getLeft()); + totalProcessTimeNanos += hsvResult.getRight(); + Imgproc.cvtColor(hsvResult.getLeft(), hsvOutputMat, Imgproc.COLOR_GRAY2BGR, 3); + procTimeStringBuilder.append(String.format("HSV: %.2fms, ", hsvResult.getRight() / 1000.0)); - /** - * Find contours from an image - * @param src the image we're looking at - * @param binaryMat a temporary image - * @param hierarchy the hierarchy of the image (just a new Mat();) - * @param emptyList a list to fill with stuff. Will be cleared - * @return the empty list, now full of contours - */ - public static List findContours(Mat src, Mat binaryMat, Mat hierarchy, List emptyList) { - src.copyTo(binaryMat); - emptyList.clear(); - Imgproc.findContours(binaryMat, emptyList, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); - binaryMat.release(); - return emptyList; - } + Pair, Long> findContoursResult = findContoursPipe.run(hsvResult.getLeft()); + totalProcessTimeNanos += findContoursResult.getRight(); + procTimeStringBuilder.append(String.format("FindContours: %.2fms, ", findContoursResult.getRight() / 1000.0)); - public static List filterContours(List inputContours, List area, List ratio, List extent, CameraValues cameraValues) { - for (MatOfPoint Contour : inputContours) { - try { - double contourArea = Imgproc.contourArea(Contour); - double AreaRatio = (contourArea / cameraValues.ImageArea) * 100; - double minArea = (MathHandler.sigmoid(area.get(0))); - double maxArea = (MathHandler.sigmoid(area.get(1))); - if (AreaRatio < minArea || AreaRatio > maxArea) { - continue; - } - var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); + Pair, Long> filterContoursResult = filterContoursPipe.run(findContoursResult.getLeft()); + totalProcessTimeNanos += filterContoursResult.getRight(); + procTimeStringBuilder.append(String.format("FilterContours: %.2fms, ", filterContoursResult.getRight() / 1000.0)); - var targetFullness = contourArea; - double minExtent = (double) (extent.get(0).doubleValue() * rect.size.area()) / 100; - double maxExtent = (double) (extent.get(1).doubleValue() * rect.size.area()) / 100; - if (targetFullness <= minExtent || contourArea >= maxExtent) { - continue; - } - Rect bb = Imgproc.boundingRect(Contour); - double aspectRatio = (bb.width / bb.height); - if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { - continue; - } - filteredContours.add(Contour); - } catch (Exception e) { - System.err.println("Error while filtering contours"); - e.printStackTrace(); - } - } - return filteredContours; - } + Pair, Long> speckleRejectResult = speckleRejectPipe.run(filterContoursResult.getLeft()); + totalProcessTimeNanos += speckleRejectResult.getRight(); + procTimeStringBuilder.append(String.format("SpeckleReject: %.2fms, ", speckleRejectResult.getRight() / 1000.0)); - @Override - Mat getOutputMat() { - return null; + Pair, Long> groupContoursResult = groupContoursPipe.run(speckleRejectResult.getLeft()); + totalProcessTimeNanos += groupContoursResult.getRight(); + procTimeStringBuilder.append(String.format("GroupContours: %.2fms, ", groupContoursResult.getRight() / 1000.0)); + + Pair, Long> sortContoursResult = sortContoursPipe.run(groupContoursResult.getLeft()); + totalProcessTimeNanos += sortContoursResult.getRight(); + procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000.0)); + + Pair, Long> collect2dTargetsResult = collect2dTargetsPipe.run(sortContoursResult.getLeft()); + totalProcessTimeNanos += collect2dTargetsResult.getRight(); + procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000.0)); + + // takes pair of (Mat of original camera image, Mat of HSV thresholded image) + Pair outputMatResult = outputMatPipe.run(Pair.of(rawCameraMat, hsvOutputMat)); + totalProcessTimeNanos += outputMatResult.getRight(); + procTimeStringBuilder.append(String.format("OutputMat: %.2fms, ", outputMatResult.getRight() / 1000.0)); + + // takes pair of (Mat to draw on, List of sorted contours) + Pair draw2dContoursResult = draw2dContoursPipe.run(Pair.of(outputMatResult.getLeft(), sortContoursResult.getLeft())); + totalProcessTimeNanos += draw2dContoursResult.getRight(); + procTimeStringBuilder.append(String.format("Draw2dContours: %.2fms, ", draw2dContoursResult.getRight() / 1000.0)); + + System.out.println(procTimeStringBuilder.toString()); + System.out.printf("Pipeline ran in %.3fms\n", totalProcessTimeNanos / 1000.0); + + return new CVPipeline2dResult(collect2dTargetsResult.getLeft(), draw2dContoursResult.getLeft()); } public static class CVPipeline2dSettings extends CVPipelineSettings { @@ -193,27 +128,20 @@ public class CVPipeline2d extends CVPipeline { double dualTargetCalibrationB = 0; } - public static class CVPipeline2dResult { - public boolean hasTarget = false; - public ArrayList targets = new ArrayList<>(); // targets sorted by likelihood - - public CVPipeline2dResult(ArrayList targets, boolean hasTarget) { + public static class CVPipeline2dResult extends CVPipelineResult { + public CVPipeline2dResult(List targets, Mat outputMat) { this.targets = targets; - this.hasTarget = hasTarget; - } - - public CVPipeline2dResult() { + this.hasTarget = !targets.isEmpty(); + this.outputMat = outputMat; } } public static class Target { - public boolean isValid = false; public double calibratedX = 0.0; public double calibratedY = 0.0; public double pitch = 0.0; public double yaw = 0.0; public double area = 0.0; - RotatedRect rawPoint; + public RotatedRect rawPoint; } - } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java index 7fce5f885..9fa8a6bd1 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java @@ -1,33 +1,33 @@ -package com.chameleonvision.classabstraction.pipeline; - -import org.opencv.core.Mat; - -public class CVPipeline3d extends CVPipeline { - - public CVPipeline3d(CVPipelineSettings settings) { - super(settings); - } - - @Override - void initPipeline() { - - } - - @Override - CVPipeline3d.CVPipeline3dResult runPipeline(Mat inputMat) { - return null; - } - - @Override - Mat getOutputMat() { - return null; - } - - public static class CVPipeline3dSettings extends CVPipelineSettings { - } - - public static class CVPipeline3dResult { - - } - -} +//package com.chameleonvision.classabstraction.pipeline; +// +//import org.opencv.core.Mat; +// +//public class CVPipeline3d extends CVPipeline { +// +// public CVPipeline3d(CVPipelineSettings settings) { +// super(settings); +// } +// +// @Override +// void initPipeline() { +// +// } +// +// @Override +// CVPipeline3d.CVPipeline3dResult runPipeline(Mat inputMat) { +// return null; +// } +// +// @Override +// Mat getOutputMat() { +// return null; +// } +// +// public static class CVPipeline3dSettings extends CVPipelineSettings { +// } +// +// public static class CVPipeline3dResult { +// +// } +// +//} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java new file mode 100644 index 000000000..eb639ea53 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java @@ -0,0 +1,11 @@ +package com.chameleonvision.classabstraction.pipeline; + +import org.opencv.core.Mat; + +import java.util.List; + +public abstract class CVPipelineResult { + List targets; + boolean hasTarget; + Mat outputMat; +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java index a0d985f40..46a82a20d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java @@ -1,25 +1,29 @@ package com.chameleonvision.classabstraction.pipeline; +import com.chameleonvision.classabstraction.camera.USBCamera; +import com.chameleonvision.vision.process.PipelineResult; import org.opencv.core.Mat; -public class DriverVisionPipeline extends CVPipeline { +public class DriverVisionPipeline extends CVPipeline { public DriverVisionPipeline(CVPipelineSettings settings) { super(settings); } @Override - void initPipeline() { - // TODO set exposure/brightness of camera + void initPipeline(USBCamera camera) { + // TODO: set camera to driver mode } @Override - Void runPipeline(Mat inputMat) { - this.outputMat = inputMat; - return null; + DriverPipelineResult runPipeline(Mat inputMat) { + return new DriverPipelineResult(inputMat); } - @Override - Mat getOutputMat() { - return this.outputMat; + public static class DriverPipelineResult extends CVPipelineResult { + public DriverPipelineResult(Mat outputMat) { + this.hasTarget = false; + this.targets = null; + outputMat.copyTo(this.outputMat); + } } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/BlurPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/BlurPipe.java new file mode 100644 index 000000000..fb2ca4057 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/BlurPipe.java @@ -0,0 +1,31 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; + +public class BlurPipe implements Pipe { + + private final int blurSize; + + private Mat outputMat = new Mat(); + + public BlurPipe(int blurSize) { + this.blurSize = blurSize; + } + + @Override + public Pair run(Mat input) { + long processStartNanos = System.nanoTime(); + + if (blurSize > 0) { + Imgproc.blur(outputMat, outputMat, new Size(blurSize, blurSize)); + } + + long processTime = processStartNanos - System.nanoTime(); + Pair output = Pair.of(outputMat, processTime); + outputMat.release(); + return output; + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java new file mode 100644 index 000000000..40c528b1a --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java @@ -0,0 +1,71 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import com.chameleonvision.classabstraction.camera.CameraStaticProperties; +import com.chameleonvision.classabstraction.pipeline.CVPipeline2d; +import com.chameleonvision.vision.CalibrationMode; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.math3.util.FastMath; +import org.opencv.core.RotatedRect; + +import java.util.ArrayList; +import java.util.List; + +public class Collect2dTargetsPipe implements Pipe, List> { + + private final CalibrationMode calibrationMode; + private final CameraStaticProperties camProps; + private final List calibrationPoint; + private final double calibrationM, calibrationB; + + private List targets = new ArrayList<>(); + + public Collect2dTargetsPipe(CalibrationMode calibrationMode, List calibrationPoint, double calibrationM, double calibrationB, CameraStaticProperties camProps) { + this.calibrationMode = calibrationMode; + this.camProps = camProps; + this.calibrationPoint = calibrationPoint; + this.calibrationM = calibrationM; + this.calibrationB = calibrationB; + } + + @Override + public Pair, Long> run(List input) { + long processStartNanos = System.nanoTime(); + + input.forEach(r -> { + CVPipeline2d.Target t = new CVPipeline2d.Target(); + t.rawPoint = r; + switch (calibrationMode) { + case None: + t.calibratedX = camProps.centerX; + t.calibratedY = camProps.centerY; + break; + case Single: + t.calibratedX = calibrationPoint.get(0).doubleValue(); + t.calibratedY = calibrationPoint.get(1).doubleValue(); + break; + case Dual: + t.calibratedX = (r.center.y - calibrationB) / calibrationM; + t.calibratedY = (r.center.x * calibrationM) + calibrationB; + break; + } + + t.pitch = calculatePitch(r.center.y, t.calibratedY); + t.yaw = calculateYaw(r.center.x, t.calibratedX); + t.area = r.size.area(); + + targets.add(t); + }); + + long processTime = processStartNanos - System.nanoTime(); + return Pair.of(targets, processTime); + } + + private double calculatePitch(double pixelY, double centerY) { + double pitch = FastMath.toDegrees(FastMath.atan((pixelY - centerY) / camProps.VerticalFocalLength)); + return (pitch * -1); + } + + private double calculateYaw(double pixelX, double centerX) { + return FastMath.toDegrees(FastMath.atan((pixelX - centerX) / camProps.HorizontalFocalLength)); + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Draw2dContoursPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Draw2dContoursPipe.java new file mode 100644 index 000000000..a970128bb --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Draw2dContoursPipe.java @@ -0,0 +1,85 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import com.chameleonvision.classabstraction.camera.CameraStaticProperties; +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.*; +import org.opencv.core.Point; +import org.opencv.imgproc.Imgproc; + +import java.awt.*; +import java.util.ArrayList; +import java.util.List; + +public class Draw2dContoursPipe implements Pipe>, Mat> { + + private final Draw2dContoursSettings settings; + private final CameraStaticProperties camProps; + + private Mat outputMat = new Mat(); + + public Draw2dContoursPipe(Draw2dContoursSettings settings, CameraStaticProperties camProps) { + this.settings = settings; + this.camProps = camProps; + } + + @Override + public Pair run(Pair> input) { + long processStartNanos = System.nanoTime(); + + input.getLeft().copyTo(outputMat); + + for (RotatedRect r : input.getRight()) { + if (r == null) continue; + + List drawnContour = new ArrayList<>(); + Point[] vertices = new Point[4]; + r.points(vertices); + MatOfPoint contour = new MatOfPoint(vertices); + drawnContour.add(contour); + + if (settings.showCentroid) { + Imgproc.circle(outputMat, r.center, 3, colorToScalar(settings.centroidColor)); + } + + if (settings.showCrosshair) { + Point xMax = new Point(camProps.centerX + 10, camProps.centerY); + Point xMin = new Point(camProps.centerX - 10, camProps.centerY); + Point yMax = new Point(camProps.centerX, camProps.centerY + 10); + Point yMin = new Point(camProps.centerX, camProps.centerY - 10); + Imgproc.line(outputMat, xMax, xMin, colorToScalar(settings.crosshairColor), 2); + Imgproc.line(outputMat, yMax, yMin, colorToScalar(settings.crosshairColor), 2); + } + + if (settings.showRotatedBox) { + Imgproc.drawContours(outputMat, drawnContour, 0, colorToScalar(settings.rotatedBoxColor), settings.boxOutlineSize); + } + + if (settings.showMaximumBox) { + Rect box = Imgproc.boundingRect(contour); + Imgproc.rectangle(outputMat, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), colorToScalar(settings.maximumBoxColor), settings.boxOutlineSize); + } + } + + long processTime = processStartNanos - System.nanoTime(); + Pair output = Pair.of(outputMat, processTime); + outputMat.release(); + return output; + } + + private Scalar colorToScalar(Color color) { + return new Scalar(color.getRed(), color.getGreen(), color.getBlue()); + } + + public static class Draw2dContoursSettings { + public boolean showCentroid = false; + public boolean showCrosshair = false; + public int boxOutlineSize = 0; + public boolean showRotatedBox = false; + public boolean showMaximumBox = false; + public Color centroidColor = Color.GREEN; + public Color crosshairColor = Color.GREEN; + public Color rotatedBoxColor = Color.BLUE; + public Color maximumBoxColor = Color.RED; + + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/ErodeDilatePipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/ErodeDilatePipe.java new file mode 100644 index 000000000..6919ca732 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/ErodeDilatePipe.java @@ -0,0 +1,38 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; + +public class ErodeDilatePipe implements Pipe { + + private final boolean erode, dilate; + private final Mat kernel; + + private Mat outputMat = new Mat(); + + public ErodeDilatePipe(boolean erode, boolean dilate, int kernelSize) { + this.erode = erode; + this.dilate = dilate; + kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(kernelSize, kernelSize)); + } + + @Override + public Pair run(Mat input) { + long processStartNanos = System.nanoTime(); + + if (erode) { + Imgproc.erode(outputMat, outputMat, kernel); + } + + if (dilate) { + Imgproc.erode(outputMat, outputMat, kernel); + } + + long processTime = processStartNanos - System.nanoTime(); + Pair output = Pair.of(outputMat, processTime); + outputMat.release(); + return output; + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FilterContoursPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FilterContoursPipe.java new file mode 100644 index 000000000..85706f842 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FilterContoursPipe.java @@ -0,0 +1,62 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import com.chameleonvision.classabstraction.camera.CameraStaticProperties; +import com.chameleonvision.util.MathHandler; +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.MatOfPoint; +import org.opencv.core.MatOfPoint2f; +import org.opencv.core.Rect; +import org.opencv.imgproc.Imgproc; + +import java.util.ArrayList; +import java.util.List; + +public class FilterContoursPipe implements Pipe, List> { + + private final List area, ratio, extent; + private final CameraStaticProperties camProps; + + private List filteredContours = new ArrayList<>(); + + public FilterContoursPipe(List area, List ratio, List extent, CameraStaticProperties camProps) { + this.area = area; + this.ratio = ratio; + this.extent = extent; + this.camProps = camProps; + } + + @Override + public Pair, Long> run(List input) { + long processStartNanos = System.nanoTime(); + + for (MatOfPoint Contour : input) { + try { + double contourArea = Imgproc.contourArea(Contour); + double AreaRatio = (contourArea / camProps.ImageArea) * 100; + double minArea = (MathHandler.sigmoid(area.get(0))); + double maxArea = (MathHandler.sigmoid(area.get(1))); + if (AreaRatio < minArea || AreaRatio > maxArea) { + continue; + } + var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); + double minExtent = (extent.get(0).doubleValue() * rect.size.area()) / 100; + double maxExtent = (extent.get(1).doubleValue() * rect.size.area()) / 100; + if (contourArea <= minExtent || contourArea >= maxExtent) { + continue; + } + Rect bb = Imgproc.boundingRect(Contour); + double aspectRatio = ((double)bb.width / bb.height); + if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { + continue; + } + filteredContours.add(Contour); + } catch (Exception e) { + System.err.println("Error while filtering contours"); + e.printStackTrace(); + } + } + + long processTime = processStartNanos - System.nanoTime(); + return Pair.of(filteredContours, processTime); + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FindContoursPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FindContoursPipe.java new file mode 100644 index 000000000..66514093b --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/FindContoursPipe.java @@ -0,0 +1,26 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint; +import org.opencv.imgproc.Imgproc; + +import java.util.ArrayList; +import java.util.List; + +public class FindContoursPipe implements Pipe> { + + private List foundContours = new ArrayList<>(); + + public FindContoursPipe() {} + + @Override + public Pair, Long> run(Mat input) { + long processStartNanos = System.nanoTime(); + + Imgproc.findContours(input, foundContours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); + + long processTime = processStartNanos - System.nanoTime(); + return Pair.of(foundContours, processTime); + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/GroupContoursPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/GroupContoursPipe.java new file mode 100644 index 000000000..e71cf9160 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/GroupContoursPipe.java @@ -0,0 +1,154 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import com.chameleonvision.util.MathHandler; +import com.chameleonvision.vision.TargetGroup; +import com.chameleonvision.vision.TargetIntersection; +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; +import org.opencv.imgproc.Moments; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class GroupContoursPipe implements Pipe, List> { + + private static final Comparator sortByMomentsX = + Comparator.comparingDouble(GroupContoursPipe::calcMomentsX); + + private final TargetGroup group; + private final TargetIntersection intersection; + + private List groupedContours = new ArrayList<>(); + private MatOfPoint2f intersectMatA = new MatOfPoint2f(); + private MatOfPoint2f intersectMatB = new MatOfPoint2f(); + + public GroupContoursPipe(TargetGroup group, TargetIntersection intersection) { + this.group = group; + this.intersection = intersection; + } + + @Override + public Pair, Long> run(List input) { + long processStartNanos = System.nanoTime(); + + List sorted = new ArrayList<>(input); + sorted.sort(sortByMomentsX); + + Collections.reverse(sorted); + + switch (group) { + case Single: { + input.forEach(c -> { + MatOfPoint2f contour = new MatOfPoint2f(); + contour.fromArray(c.toArray()); + if (contour.cols() != 0 && contour.rows() != 0) { + RotatedRect rect = Imgproc.minAreaRect(contour); + groupedContours.add(rect); + } + }); + break; + } + case Dual: { + for (var i = 0; i < input.size(); i++) { + List finalContourList = new ArrayList<>(input.get(i).toList()); + + try { + MatOfPoint firstContour = input.get(i); + MatOfPoint secondContour = input.get(i + 1); + + if (isIntersecting(firstContour, secondContour)) { + finalContourList.addAll(secondContour.toList()); + } else { + finalContourList.clear(); + continue; + } + + firstContour.release(); + secondContour.release(); + + MatOfPoint2f contour = new MatOfPoint2f(); + contour.fromList(finalContourList); + + if (contour.cols() != 0 && contour.rows() != 0) { + RotatedRect rect = Imgproc.minAreaRect(contour); + groupedContours.add(rect); + } + } catch (IndexOutOfBoundsException e) { + System.err.println("GroupContours: WTF"); + finalContourList.clear(); + } + } + break; + } + } + + long processTime = processStartNanos - System.nanoTime(); + return Pair.of(groupedContours, processTime); + } + + private static double calcMomentsX(MatOfPoint c) { + Moments m = Imgproc.moments(c); + return (m.get_m10() / m.get_m00()); + } + + private boolean isIntersecting(MatOfPoint contourOne, MatOfPoint contourTwo) { + if (intersection.equals(TargetIntersection.None)) { + return true; + } + + try { + intersectMatA.fromArray(contourOne.toArray()); + intersectMatB.fromArray(contourTwo.toArray()); + RotatedRect a = Imgproc.fitEllipse(intersectMatA); + RotatedRect b = Imgproc.fitEllipse(intersectMatB); + double mA = MathHandler.toSlope(a.angle); + double mB = MathHandler.toSlope(b.angle); + double x0A = a.center.x; + double y0A = a.center.y; + double x0B = b.center.x; + double y0B = b.center.y; + double intersectionX = ((mA * x0A) - y0A - (mB * x0B) + y0B) / (mA - mB); + double intersectionY = (mA * (intersectionX - x0A)) + y0A; + double massX = (x0A + x0B) / 2; + double massY = (y0A + y0B) / 2; + switch (intersection) { + case Up: { + if (intersectionY < massY) { + if (mA > 0 && mB < 0) { + return true; + } + } + break; + } + case Down: { + if (intersectionY > massY) { + if (mA < 0 && mB > 0) { + return true; + } + } + + break; + } + case Left: { + if (intersectionX < massX) { + + return true; + } + break; + } + case Right: { + if (intersectionX > massX) { + return true; + } + break; + } + } + return false; + } catch (Exception e) { + return false; + } + } +} \ No newline at end of file diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/HsvPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/HsvPipe.java new file mode 100644 index 000000000..f836e17d5 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/HsvPipe.java @@ -0,0 +1,35 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.Scalar; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; + +public class HsvPipe implements Pipe { + + private final Scalar hsvLower, hsvUpper; + + private Mat outputMat = new Mat(); + + public HsvPipe(Scalar hsvLower, Scalar hsvUpper) { + this.hsvLower = hsvLower; + this.hsvUpper = hsvUpper; + } + + @Override + public Pair run(Mat input) { + long processStartNanos = System.nanoTime(); + + Imgproc.cvtColor(input, outputMat, Imgproc.COLOR_RGB2HSV, 3); + + Core.inRange(outputMat, hsvLower, hsvUpper, outputMat); + + long processTime = processStartNanos - System.nanoTime(); + Pair output = Pair.of(outputMat, processTime); + outputMat.release(); + return output; + } +} + diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/OutputMatPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/OutputMatPipe.java new file mode 100644 index 000000000..5f413860c --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/OutputMatPipe.java @@ -0,0 +1,31 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; + +public class OutputMatPipe implements Pipe, Mat> { + + private boolean showThresholded; + + private Mat outputMat = new Mat(); + + public OutputMatPipe(boolean showThresholded) { + this.showThresholded = showThresholded; + } + + @Override + public Pair run(Pair input) { + long processStartNanos = System.nanoTime(); + + if (showThresholded) { + input.getRight().copyTo(outputMat); + } else { + input.getLeft().copyTo(outputMat); + } + + long processTime = processStartNanos - System.nanoTime(); + Pair output = Pair.of(outputMat, processTime); + outputMat.release(); + return output; + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Pipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Pipe.java new file mode 100644 index 000000000..7d629b29f --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Pipe.java @@ -0,0 +1,13 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import org.apache.commons.lang3.tuple.Pair; + +public interface Pipe { + /** + * + * @param input Input object for pipe + * @return Returns a Pair containing the process time in Nanoseconds, + * and the output object + */ + Pair run(I input); +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/RotateFlipPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/RotateFlipPipe.java new file mode 100644 index 000000000..569cc0bf6 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/RotateFlipPipe.java @@ -0,0 +1,33 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import com.chameleonvision.vision.ImageFlipMode; +import com.chameleonvision.vision.ImageRotation; +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Core; +import org.opencv.core.Mat; + +public class RotateFlipPipe implements Pipe { + + private final ImageRotation rotation; + private final ImageFlipMode flip; + + private Mat outputMat = new Mat(); + + public RotateFlipPipe(ImageRotation rotation, ImageFlipMode flip) { + this.rotation = rotation; + this.flip = flip; + } + + @Override + public Pair run(Mat input) { + long processStartNanos = System.nanoTime(); + + Core.flip(input, outputMat, flip.value); + Core.rotate(outputMat, outputMat, rotation.value); + + long processTime = processStartNanos - System.nanoTime(); + Pair output = Pair.of(outputMat, processTime); + outputMat.release(); + return output; + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SortContoursPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SortContoursPipe.java new file mode 100644 index 000000000..46f7f5fa3 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SortContoursPipe.java @@ -0,0 +1,74 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import com.chameleonvision.classabstraction.camera.CameraStaticProperties; +import com.chameleonvision.vision.SortMode; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.math3.util.FastMath; +import org.opencv.core.RotatedRect; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +public class SortContoursPipe implements Pipe, List> { + + private final Comparator SortByCentermostComparator = Comparator.comparingDouble(this::calcCenterDistance); + + private static final Comparator SortByLargestComparator = (rect1, rect2) -> Double.compare(rect2.size.area(), rect1.size.area()); + private static final Comparator SortBySmallestComparator = SortByLargestComparator.reversed(); + + private static final Comparator SortByHighestComparator = (rect1, rect2) -> Double.compare(rect2.center.y, rect1.center.y); + private static final Comparator SortByLowestComparator = SortByHighestComparator.reversed(); + + private static final Comparator SortByLeftmostComparator = Comparator.comparingDouble(rect -> rect.center.x); + private static final Comparator SortByRightmostComparator = SortByLeftmostComparator.reversed(); + + + private final SortMode sort; + private final CameraStaticProperties camProps; + + private List sortedContours = new ArrayList<>(); + + public SortContoursPipe(SortMode sort, CameraStaticProperties camProps) { + this.sort = sort; + this.camProps = camProps; + } + + @Override + public Pair, Long> run(List input) { + long processStartNanos = System.nanoTime(); + + switch (sort) { + case Largest: + input.sort(SortByLargestComparator); + break; + case Smallest: + input.sort(SortBySmallestComparator); + break; + case Highest: + input.sort(SortByHighestComparator); + break; + case Lowest: + input.sort(SortByLowestComparator); + break; + case Leftmost: + input.sort(SortByLeftmostComparator); + break; + case Rightmost: + input.sort(SortByRightmostComparator); + break; + case Centermost: + input.sort(SortByCentermostComparator); + break; + default: + break; + } + + long processTime = processStartNanos - System.nanoTime(); + return Pair.of(sortedContours, processTime); + } + + private double calcCenterDistance(RotatedRect rect) { + return FastMath.sqrt(FastMath.pow(camProps.centerX - rect.center.x, 2) + FastMath.pow(camProps.centerY - rect.center.y, 2)); + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SpeckleRejectPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SpeckleRejectPipe.java new file mode 100644 index 000000000..309e0ce22 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/SpeckleRejectPipe.java @@ -0,0 +1,44 @@ +package com.chameleonvision.classabstraction.pipeline.pipes; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint; +import org.opencv.imgproc.Imgproc; + +import java.util.ArrayList; +import java.util.List; + +public class SpeckleRejectPipe implements Pipe, List> { + + private final double minPercentOfAvg; + + private List despeckledContours = new ArrayList<>(); + + public SpeckleRejectPipe(double minPercentOfAvg) { + this.minPercentOfAvg = minPercentOfAvg; + } + + @Override + public Pair, Long> run(List input) { + long processStartNanos = System.nanoTime(); + + double averageArea = 0.0; + + for (MatOfPoint c : input) { + averageArea += Imgproc.contourArea(c); + } + + averageArea /= input.size(); + + double minAllowedArea = minPercentOfAvg / 100.0 * averageArea; + + for (MatOfPoint c : input) { + if (Imgproc.contourArea(c) >= minAllowedArea) { + despeckledContours.add(c); + } + } + + long processTime = processStartNanos - System.nanoTime(); + return Pair.of(despeckledContours, processTime); + } +} diff --git a/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java b/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java index 5a778bd17..d29fd4cf4 100644 --- a/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java +++ b/Main/src/main/java/com/chameleonvision/vision/ImageFlipMode.java @@ -1,5 +1,14 @@ package com.chameleonvision.vision; public enum ImageFlipMode { - NONE, VERTICAL, HORIZONTAL, BOTH + NONE(Integer.MIN_VALUE), + VERTICAL(1), + HORIZONTAL(0), + BOTH(-1); + + public final int value; + + ImageFlipMode(int value) { + this.value = value; + } } diff --git a/Main/src/main/java/com/chameleonvision/vision/ImageRotation.java b/Main/src/main/java/com/chameleonvision/vision/ImageRotation.java new file mode 100644 index 000000000..182122409 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/ImageRotation.java @@ -0,0 +1,16 @@ +package com.chameleonvision.vision; + +import org.opencv.core.Core; + +public enum ImageRotation { + DEG_0(-1), + DEG_90(Core.ROTATE_90_CLOCKWISE), + DEG_180(Core.ROTATE_180), + DEG_270(Core.ROTATE_90_COUNTERCLOCKWISE); + + public final int value; + + ImageRotation(int value) { + this.value = value; + } +} diff --git a/Main/src/main/java/com/chameleonvision/vision/TargetGroup.java b/Main/src/main/java/com/chameleonvision/vision/TargetGroup.java index 278e53852..30574af9c 100644 --- a/Main/src/main/java/com/chameleonvision/vision/TargetGroup.java +++ b/Main/src/main/java/com/chameleonvision/vision/TargetGroup.java @@ -1,5 +1,6 @@ package com.chameleonvision.vision; public enum TargetGroup { - Single,Dual + Single, + Dual } From 338c59e783fd9d4320b5215535f62d098165c68d Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 7 Nov 2019 09:02:23 -0800 Subject: [PATCH 09/84] Start working on USBCamera --- Main/chameleon-vision.iml | 13 ++++++ .../camera/CameraProcess.java | 2 +- .../classabstraction/camera/USBCamera.java | 40 +++++++++++++++++-- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 1799ea932..6f10ef5f5 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -12,6 +12,19 @@ + + + + + + + + + + + + + diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java index fe5777c0c..df4a4c38f 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java @@ -4,7 +4,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; public interface CameraProcess { - Pair getFrame(); + Pair getFrame(Mat frame); void setExposure(int exposure); void setBrightness(int brightness); diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java index 7175ff7f9..15dc844ad 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java @@ -1,15 +1,49 @@ package com.chameleonvision.classabstraction.camera; +import edu.wpi.cscore.CvSink; import edu.wpi.cscore.UsbCamera; +import edu.wpi.cscore.VideoException; import edu.wpi.cscore.VideoMode; +import edu.wpi.first.cameraserver.CameraServer; +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; -public class USBCamera { +public class USBCamera implements CameraProcess { private final UsbCamera baseCamera; public final CameraProperties properties; + private final CvSink cvSink; + private Mat imageBuffer = new Mat(); - public USBCamera(UsbCamera camera) { + public USBCamera(UsbCamera camera, double fov) { baseCamera = camera; + cvSink = CameraServer.getInstance().getVideo(baseCamera); VideoMode vidMode = new VideoMode(VideoMode.PixelFormat.kYUYV, 640, 480, 60); - properties = new CameraProperties(baseCamera, 75); + properties = new CameraProperties(baseCamera, fov); + } + + @Override + public Pair getFrame(Mat frame) { + var timestamp = System.nanoTime(); + cvSink.grabFrame(imageBuffer); + imageBuffer.copyTo(frame); + return Pair.of(frame, timestamp - System.nanoTime()); + } + + @Override + public void setExposure(int exposure) { + try { + baseCamera.setExposureManual(exposure); + } catch (VideoException e) { + System.err.println("USBCamera Does not support exposure change"); + } + } + + @Override + public void setBrightness(int brightness) { + try { + baseCamera.setBrightness(brightness); + } catch (VideoException e) { + System.err.println("USBCamera Does not support brightness change"); + } } } From cc2c065dc8352f9313f0d4c5bd0258defd97735a Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 7 Nov 2019 09:10:09 -0800 Subject: [PATCH 10/84] Rename USBCamera to USBCameraProcess & abstractify - Most classes now take a CameraProcess instead of a USBCameraProcess --- .../classabstraction/camera/CameraProcess.java | 2 ++ .../{USBCamera.java => USBCameraProcess.java} | 15 ++++++++++----- .../classabstraction/pipeline/CVPipeline.java | 4 ++-- .../classabstraction/pipeline/CVPipeline2d.java | 10 +++++----- .../pipeline/DriverVisionPipeline.java | 5 ++--- .../chameleonvision/vision/camera/USBCamera.java | 6 +++--- .../vision/process/StandardCVProcess.java | 2 +- .../vision/process/USBCameraProcess.java | 2 +- .../vision/process/VisionProcess.java | 2 +- 9 files changed, 27 insertions(+), 21 deletions(-) rename Main/src/main/java/com/chameleonvision/classabstraction/camera/{USBCamera.java => USBCameraProcess.java} (75%) diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java index df4a4c38f..d8474979c 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java @@ -4,6 +4,8 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; public interface CameraProcess { + CameraProperties getProperties(); + Pair getFrame(Mat frame); void setExposure(int exposure); diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java similarity index 75% rename from Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java rename to Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java index 15dc844ad..fcaea50e6 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCamera.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java @@ -8,19 +8,24 @@ import edu.wpi.first.cameraserver.CameraServer; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; -public class USBCamera implements CameraProcess { +public class USBCameraProcess implements CameraProcess { private final UsbCamera baseCamera; - public final CameraProperties properties; private final CvSink cvSink; private Mat imageBuffer = new Mat(); + public final CameraProperties properties; - public USBCamera(UsbCamera camera, double fov) { + public USBCameraProcess(UsbCamera camera, double fov) { baseCamera = camera; cvSink = CameraServer.getInstance().getVideo(baseCamera); VideoMode vidMode = new VideoMode(VideoMode.PixelFormat.kYUYV, 640, 480, 60); properties = new CameraProperties(baseCamera, fov); } + @Override + public CameraProperties getProperties() { + return properties; + } + @Override public Pair getFrame(Mat frame) { var timestamp = System.nanoTime(); @@ -34,7 +39,7 @@ public class USBCamera implements CameraProcess { try { baseCamera.setExposureManual(exposure); } catch (VideoException e) { - System.err.println("USBCamera Does not support exposure change"); + System.err.println("USBCameraProcess Does not support exposure change"); } } @@ -43,7 +48,7 @@ public class USBCamera implements CameraProcess { try { baseCamera.setBrightness(brightness); } catch (VideoException e) { - System.err.println("USBCamera Does not support brightness change"); + System.err.println("USBCameraProcess Does not support brightness change"); } } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java index b0a77c234..e7c686fc1 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java @@ -1,6 +1,6 @@ package com.chameleonvision.classabstraction.pipeline; -import com.chameleonvision.classabstraction.camera.USBCamera; +import com.chameleonvision.classabstraction.camera.CameraProcess; import org.opencv.core.Mat; /** @@ -16,6 +16,6 @@ public abstract class CVPipeline { - private USBCamera camera; + private CameraProcess cameraProcess; private Mat rawCameraMat = new Mat(); private Mat hsvOutputMat = new Mat(); @@ -23,8 +23,8 @@ public class CVPipeline2d extends CVPipeline { @@ -10,7 +9,7 @@ public class DriverVisionPipeline extends CVPipeline v.fps >= MINIMUM_FPS && v.width >= MINIMUM_WIDTH && v.height >= MINIMUM_HEIGHT && ALLOWED_PIXEL_FORMATS.contains(v.pixelFormat)).toArray(VideoMode[]::new); if (availableVideoModes.length == 0) { - System.err.println("USBCamera not supported!"); + System.err.println("USBCameraProcess not supported!"); throw new RuntimeException(new CameraException(CameraException.CameraExceptionType.BAD_CAMERA)); } if (videoModeIndex <= availableVideoModes.length - 1) { @@ -283,7 +283,7 @@ public class USBCamera { try { UsbCam.setExposureManual(exposure); } catch (VideoException e) { - System.err.println("USBCamera Does not support exposure change"); + System.err.println("USBCameraProcess Does not support exposure change"); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java index e7a0585b4..195ca8771 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/StandardCVProcess.java @@ -122,7 +122,7 @@ public class StandardCVProcess implements CVProcess { pipelineResult.IsValid = true; switch (currentPipeline.calibrationMode) { case None: - ///use the center of the USBCamera to find the pitch and yaw difference + ///use the center of the USBCameraProcess to find the pitch and yaw difference pipelineResult.CalibratedX = cameraValues.CenterX; pipelineResult.CalibratedY = cameraValues.CenterY; break; diff --git a/Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java index 3491c893a..b01b24942 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/USBCameraProcess.java @@ -80,7 +80,7 @@ public class USBCameraProcess implements CameraProcess { } } - // USBCamera stuff + // USBCameraProcess stuff @Override public String getCamName() { diff --git a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java index b8a4fa360..44caa86f9 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java @@ -41,7 +41,7 @@ public class VisionProcess implements Runnable { public VisionProcess(CameraProcess cameraProcess) { - // USBCamera settings + // USBCameraProcess settings cvProcess = new StandardCVProcess(cameraProcess.getCamVals()); this.cameraProcess = cameraProcess; // new USBCameraProcess(cameraProcess); From 1e7f380f07236a47454e5fd95186dd9b5fafe9fa Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 7 Nov 2019 16:53:31 -0800 Subject: [PATCH 11/84] Rename StreamDivisor enum and work on CameraStreamer --- .../camera/CameraProcess.java | 14 +++++ .../camera/CameraProperties.java | 6 +- .../camera/CameraStaticProperties.java | 32 +++++------ .../camera/CameraStreamer.java | 55 ++++++++++++++++++- .../camera/USBCameraProcess.java | 10 ++-- .../pipeline/pipes/Collect2dTargetsPipe.java | 4 +- .../pipeline/pipes/FilterContoursPipe.java | 2 +- .../vision/camera/CameraDeserializer.java | 2 +- .../vision/camera/StreamDivisor.java | 8 +-- .../vision/camera/USBCamera.java | 2 +- 10 files changed, 101 insertions(+), 34 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java index d8474979c..bab9235fd 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProcess.java @@ -6,8 +6,22 @@ import org.opencv.core.Mat; public interface CameraProcess { CameraProperties getProperties(); + /** + * Get the next camera frame + * @param frame the frame to copy the image into + * @return a Pair of the captured image and how long it took to grab the frame (in uS) + */ Pair getFrame(Mat frame); + /** + * Set the exposure of the camera + * @param exposure the new exposure to set the camera to + */ void setExposure(int exposure); + + /** + * Set the exposure of the camera + * @param brightness the new brightness to set the camera to + */ void setBrightness(int brightness); } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java index 4e0ab8a4d..2d7589d5d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraProperties.java @@ -21,7 +21,7 @@ public class CameraProperties { private static final List ALLOWED_PIXEL_FORMATS = Arrays.asList(VideoMode.PixelFormat.kYUYV, VideoMode.PixelFormat.kMJPEG); private static final Predicate kMinFPSPredicate = (videoMode -> videoMode.fps >= MINIMUM_FPS); - private static final Predicate kMinSizePredicate = (videoMode -> videoMode.width >= MINIMUM_FPS && videoMode.height >= MINIMUM_FPS); + private static final Predicate kMinSizePredicate = (videoMode -> videoMode.width >= MINIMUM_WIDTH && videoMode.height >= MINIMUM_HEIGHT); private static final Predicate kPixelFormatPredicate = (videoMode -> ALLOWED_PIXEL_FORMATS.contains(videoMode.pixelFormat)); public CameraStaticProperties staticProperties; @@ -46,11 +46,11 @@ public class CameraProperties { } public double CalculatePitch(double PixelY, double centerY) { - double pitch = FastMath.toDegrees(FastMath.atan((PixelY - centerY) / staticProperties.VerticalFocalLength)); + double pitch = FastMath.toDegrees(FastMath.atan((PixelY - centerY) / staticProperties.verticalFocalLength)); return (pitch * -1); } public double CalculateYaw(double PixelX, double centerX) { - return FastMath.toDegrees(FastMath.atan((PixelX - centerX) / staticProperties.HorizontalFocalLength)); + return FastMath.toDegrees(FastMath.atan((PixelX - centerX) / staticProperties.horizontalFocalLength)); } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java index 5f9e9faee..d31b582c0 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStaticProperties.java @@ -5,32 +5,32 @@ import org.apache.commons.math3.util.FastMath; public class CameraStaticProperties { - public final int ImageWidth; - public final int ImageHeight; - public final double FOV; - public final double ImageArea; + public final int imageWidth; + public final int imageHeight; + public final double fov; + public final double imageArea; public final double centerX; public final double centerY; - public final double HorizontalFocalLength; - public final double VerticalFocalLength; + public final double horizontalFocalLength; + public final double verticalFocalLength; public CameraStaticProperties(int imageWidth, int imageHeight, double fov) { - ImageWidth = imageWidth; - ImageHeight = imageHeight; - FOV = fov; - ImageArea = ImageWidth * ImageHeight; - centerX = ((double) ImageWidth / 2) - 0.5; - centerY = ((double) ImageHeight / 2) - 0.5; + this.imageWidth = imageWidth; + this.imageHeight = imageHeight; + this.fov = fov; + imageArea = this.imageWidth * this.imageHeight; + centerX = ((double) this.imageWidth / 2) - 0.5; + centerY = ((double) this.imageHeight / 2) - 0.5; // pinhole model calculations - double diagonalView = FastMath.toRadians(FOV); - Fraction aspectFraction = new Fraction(ImageWidth, ImageHeight); + double diagonalView = FastMath.toRadians(this.fov); + Fraction aspectFraction = new Fraction(this.imageWidth, this.imageHeight); int horizontalRatio = aspectFraction.getNumerator(); int verticalRatio = aspectFraction.getDenominator(); double diagonalAspect = FastMath.hypot(horizontalRatio, verticalRatio); double horizontalView = FastMath.atan(FastMath.tan(diagonalView / 2) * (horizontalRatio / diagonalAspect)) * 2; double verticalView = FastMath.atan(FastMath.tan(diagonalView / 2) * (verticalRatio / diagonalAspect)) * 2; - HorizontalFocalLength = ImageWidth / (2 * FastMath.tan(horizontalView /2)); - VerticalFocalLength = ImageHeight / (2 * FastMath.tan(verticalView /2)); + horizontalFocalLength = this.imageWidth / (2 * FastMath.tan(horizontalView /2)); + verticalFocalLength = this.imageHeight / (2 * FastMath.tan(verticalView /2)); } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java index fc3dc2a16..2f98f37c8 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraStreamer.java @@ -1,7 +1,58 @@ package com.chameleonvision.classabstraction.camera; +import com.chameleonvision.vision.camera.CameraManager; +import com.chameleonvision.vision.camera.StreamDivisor; +import com.chameleonvision.web.ServerHandler; +import edu.wpi.cscore.CvSource; +import edu.wpi.first.cameraserver.CameraServer; +import org.opencv.core.CvType; import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; + +public class CameraStreamer { + + private final CameraProcess cameraProcess; + private final String name; + private StreamDivisor divisor = StreamDivisor.NONE; + private CvSource cvSource; + private final Object streamBufferLock = new Object(); + private Mat streamBuffer = new Mat(); + + public CameraStreamer(CameraProcess cameraProcess, String name) { + this.cameraProcess = cameraProcess; + this.name = name; + this.cvSource = CameraServer.getInstance().putVideo(name, + cameraProcess.getProperties().staticProperties.imageWidth / divisor.value, + cameraProcess.getProperties().staticProperties.imageHeight / divisor.value); + } + + + public void setDivisor(StreamDivisor newDivisor) { + this.divisor = newDivisor; + var camValues = cameraProcess.getProperties(); + var newWidth = camValues.staticProperties.imageWidth / newDivisor.value; + var newHeight = camValues.staticProperties.imageHeight / newDivisor.value; + synchronized (streamBufferLock) { + this.streamBuffer = new Mat(newWidth, newHeight, CvType.CV_8UC3); + this.cvSource = CameraServer.getInstance().putVideo(this.name, + cameraProcess.getProperties().staticProperties.imageWidth / divisor.value, + cameraProcess.getProperties().staticProperties.imageHeight / divisor.value); + } + ServerHandler.sendFullSettings(); + } + + public void runStream() { + var newFrame = cameraProcess.getFrame(streamBuffer); + var image = newFrame.getLeft(); + if (divisor.value != 1) { + var camVal = cameraProcess.getProperties().staticProperties; + var newWidth = camVal.imageWidth / divisor.value; + var newHeight = camVal.imageHeight / divisor.value; + Size newSize = new Size(newWidth, newHeight); + Imgproc.resize(image, image, newSize); + } + cvSource.putFrame(image); + } -public interface CameraStreamer { - void streamFrame(Mat frame); } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java index fcaea50e6..37b259315 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java @@ -28,10 +28,12 @@ public class USBCameraProcess implements CameraProcess { @Override public Pair getFrame(Mat frame) { - var timestamp = System.nanoTime(); - cvSink.grabFrame(imageBuffer); - imageBuffer.copyTo(frame); - return Pair.of(frame, timestamp - System.nanoTime()); + Long deltaTime; + synchronized (cvSink) { + deltaTime = cvSink.grabFrame(imageBuffer) * 1000L; + imageBuffer.copyTo(frame); + } + return Pair.of(frame, deltaTime); } @Override diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java index 40c528b1a..abf90f969 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/pipes/Collect2dTargetsPipe.java @@ -61,11 +61,11 @@ public class Collect2dTargetsPipe implements Pipe, List, List maxArea) { diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java index 50bf15ecd..06cf6a539 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraDeserializer.java @@ -31,7 +31,7 @@ public class CameraDeserializer implements JsonDeserializer { boolean isDriver = isDriverObj != null && isDriverObj.getAsBoolean(); int driverExposure = driverExposureObj == null ? USBCamera.DEFAULT_EXPOSURE : driverExposureObj.getAsInt(); int driverBrightness = driverBrightnessObj == null ? USBCamera.DEFAULT_BRIGHTNESS : driverBrightnessObj.getAsInt(); - StreamDivisor divisor = divisorObj == null ? StreamDivisor.none : StreamDivisor.values()[divisorObj.getAsInt()]; + StreamDivisor divisor = divisorObj == null ? StreamDivisor.NONE : StreamDivisor.values()[divisorObj.getAsInt()]; var pipelines = jsonObj.get("pipelines"); List actualPipelines = new ArrayList<>(); diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/StreamDivisor.java b/Main/src/main/java/com/chameleonvision/vision/camera/StreamDivisor.java index 8edb165e2..f621945b4 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/StreamDivisor.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/StreamDivisor.java @@ -1,10 +1,10 @@ package com.chameleonvision.vision.camera; public enum StreamDivisor { - none(1), - half(2), - quarter(4), - sixth(6); + NONE(1), + HALF(2), + QUARTER(4), + SIXTH(6); public final Integer value; diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCamera.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCamera.java index 726a1712e..28f2db192 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCamera.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCamera.java @@ -16,7 +16,7 @@ import java.util.stream.IntStream; public class USBCamera { private static final double DEFAULT_FOV = 60.8; - private static final StreamDivisor DEFAULT_STREAMDIVISOR = StreamDivisor.none; + private static final StreamDivisor DEFAULT_STREAMDIVISOR = StreamDivisor.NONE; public static final int DEFAULT_EXPOSURE = 50; public static final int DEFAULT_BRIGHTNESS = 50; private static final int MINIMUM_FPS = 30; From 80f711c37c5cf169addb5375894e6afd519dbdfe Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 10 Nov 2019 11:47:56 -0500 Subject: [PATCH 12/84] Begin work on VisionProcess, refinement of CVPipeline interactions --- .../classabstraction/VisionManager.java | 61 +++++++++++++++++ .../classabstraction/VisionProcess.java | 41 +++++++++++ .../classabstraction/camera/CameraConfig.java | 10 +++ .../camera/USBCameraProcess.java | 8 +-- .../classabstraction/pipeline/CVPipeline.java | 22 ++++-- .../pipeline/CVPipeline2d.java | 29 +++----- .../pipeline/CVPipeline2dSettings.java | 29 ++++++++ .../pipeline/CVPipeline3d.java | 66 +++++++++--------- .../pipeline/CVPipeline3dSettings.java | 5 ++ .../pipeline/CVPipelineResult.java | 12 +++- .../pipeline/CVPipelineSettings.java | 29 ++------ .../pipeline/DriverVisionPipeline.java | 30 +++++--- .../vision/process/VisionProcess.java | 68 ++++++++++++++++--- 13 files changed, 300 insertions(+), 110 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java b/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java new file mode 100644 index 000000000..e5d735f2f --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java @@ -0,0 +1,61 @@ +package com.chameleonvision.classabstraction; + +import com.chameleonvision.classabstraction.camera.USBCameraProcess; +import com.chameleonvision.settings.SettingsManager; +import com.chameleonvision.util.FileHelper; +import com.chameleonvision.vision.camera.CameraDeserializer; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import edu.wpi.cscore.UsbCamera; +import edu.wpi.cscore.UsbCameraInfo; +import org.opencv.videoio.VideoCapture; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; + +public class VisionManager { + + private static final Path CamConfigPath = Paths.get(SettingsManager.SettingsPath.toString(), "cameras"); + + public static final LinkedHashMap UsbCameraInfosByCameraName = new LinkedHashMap<>(); + public static final LinkedHashMap VisionProcessesByCameraName = new LinkedHashMap<>(); + + + public static boolean initializeSources() { + int suffix = 0; + for (UsbCameraInfo info : UsbCamera.enumerateUsbCameras()) { + VideoCapture cap = new VideoCapture(info.dev); + if (cap.isOpened()) { + cap.release(); + String name = info.name; + while (UsbCameraInfosByCameraName.containsKey(name)) { + suffix++; + name = String.format("%s (%d)", name, suffix); + } + UsbCameraInfosByCameraName.put(name, info); + } + } + + if (UsbCameraInfosByCameraName.isEmpty()) { + return false; + } + + FileHelper.CheckPath(CamConfigPath); + UsbCameraInfosByCameraName.forEach((cameraName, cameraInfo) -> { + Path cameraConfigPath = Paths.get(CamConfigPath.toString(), String.format("%s.json", cameraName)); + File cameraConfigFile = new File(cameraConfigPath.toString()); + if (cameraConfigFile.exists() && cameraConfigFile.length() != 0) { + try { + Gson gson = new GsonBuilder()().registerTypeAdapter(USBCameraProcess.class, new CameraDeserializer()) + } + } + }) + + } + + public static void initializeProcesses() { + + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java new file mode 100644 index 000000000..eb54fcf7c --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/VisionProcess.java @@ -0,0 +1,41 @@ +package com.chameleonvision.classabstraction; + +import com.chameleonvision.classabstraction.camera.CameraProcess; +import com.chameleonvision.classabstraction.pipeline.CVPipeline; +import com.chameleonvision.classabstraction.pipeline.CVPipelineSettings; +import com.chameleonvision.classabstraction.pipeline.DriverVisionPipeline; + +import java.util.ArrayList; +import java.util.List; + +public class VisionProcess { + + private final CameraProcess cameraProcess; + private final List pipelines = new ArrayList<>(); + private CVPipeline currentPipeline; + + private final CVPipelineSettings driverVisionSettings = new CVPipelineSettings(); + + public VisionProcess(CameraProcess cameraProcess) { + this.cameraProcess = cameraProcess; + + pipelines.add(new DriverVisionPipeline(() -> driverVisionSettings)); + setPipeline(pipelines.get(0)); + } + + public void setPipeline(int pipelineIndex) { + CVPipeline newPipeline = pipelines.get(pipelineIndex); + if (newPipeline != null) { + setPipeline(newPipeline); + } + } + + public void setPipeline(CVPipeline pipeline) { + currentPipeline = pipeline; + currentPipeline.initPipeline(cameraProcess); + } + + public CVPipeline getCurrentPipeline() { + return currentPipeline; + } +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java new file mode 100644 index 000000000..830b9a358 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java @@ -0,0 +1,10 @@ +package com.chameleonvision.classabstraction.camera; + +public class CameraConfig { + public double FOV; + public String Path; + public String Name; + public String Nickname; + public com.chameleonvision.vision.camera.StreamDivisor StreamDivisor; +// public +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java index 37b259315..be5b506ef 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java @@ -14,11 +14,11 @@ public class USBCameraProcess implements CameraProcess { private Mat imageBuffer = new Mat(); public final CameraProperties properties; - public USBCameraProcess(UsbCamera camera, double fov) { + public USBCameraProcess(UsbCamera camera, CameraConfig config) { baseCamera = camera; cvSink = CameraServer.getInstance().getVideo(baseCamera); VideoMode vidMode = new VideoMode(VideoMode.PixelFormat.kYUYV, 640, 480, 60); - properties = new CameraProperties(baseCamera, fov); + properties = new CameraProperties(baseCamera, config.FOV); } @Override @@ -41,7 +41,7 @@ public class USBCameraProcess implements CameraProcess { try { baseCamera.setExposureManual(exposure); } catch (VideoException e) { - System.err.println("USBCameraProcess Does not support exposure change"); + System.err.println("Current camera does not support exposure change"); } } @@ -50,7 +50,7 @@ public class USBCameraProcess implements CameraProcess { try { baseCamera.setBrightness(brightness); } catch (VideoException e) { - System.err.println("USBCameraProcess Does not support brightness change"); + System.err.println("Current camera does not support brightness change"); } } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java index e7c686fc1..3706bf812 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline.java @@ -3,19 +3,29 @@ package com.chameleonvision.classabstraction.pipeline; import com.chameleonvision.classabstraction.camera.CameraProcess; import org.opencv.core.Mat; +import java.util.function.Supplier; + /** * * @param Pipeline result type */ public abstract class CVPipeline { - protected S settings; - private Mat inputMat; protected Mat outputMat; + CameraProcess cameraProcess; + final Supplier settingsSupplier; - public CVPipeline(S settings) { - this.settings = settings; + public CVPipeline(Supplier settingsSupplier) { + this.settingsSupplier = settingsSupplier; } - abstract void initPipeline(CameraProcess camera); - abstract R runPipeline(Mat inputMat); + public S getSettings() { + return settingsSupplier.get(); + } + + public void initPipeline(CameraProcess camera) { + cameraProcess = camera; + cameraProcess.setExposure((int) getSettings().exposure); + cameraProcess.setBrightness((int) getSettings().brightness); + } + abstract public R runPipeline(Mat inputMat); } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java index bba5566c0..8851a5422 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2d.java @@ -1,6 +1,5 @@ package com.chameleonvision.classabstraction.pipeline; -import com.chameleonvision.classabstraction.camera.CameraProcess; import com.chameleonvision.classabstraction.camera.CameraStaticProperties; import com.chameleonvision.classabstraction.pipeline.pipes.*; import com.chameleonvision.vision.ImageRotation; @@ -9,29 +8,26 @@ import org.opencv.core.*; import org.opencv.imgproc.Imgproc; import java.util.List; +import java.util.function.Supplier; + +import static com.chameleonvision.classabstraction.pipeline.CVPipeline2d.*; @SuppressWarnings("WeakerAccess") -public class CVPipeline2d extends CVPipeline { - - private CameraProcess cameraProcess; +public class CVPipeline2d extends CVPipeline { private Mat rawCameraMat = new Mat(); private Mat hsvOutputMat = new Mat(); - public CVPipeline2d(CVPipeline2dSettings settings) { - super(settings); + public CVPipeline2d(Supplier settingsSupplier) { + super(settingsSupplier); } @Override - void initPipeline(CameraProcess cam) { - cameraProcess = cam; - } - - @Override - CVPipeline2d.CVPipeline2dResult runPipeline(Mat inputMat) { + public CVPipeline2dResult runPipeline(Mat inputMat) { long totalProcessTimeNanos = 0; StringBuilder procTimeStringBuilder = new StringBuilder(); + var settings = settingsSupplier.get(); CameraStaticProperties camProps = cameraProcess.getProperties().staticProperties; inputMat.copyTo(rawCameraMat); @@ -123,16 +119,9 @@ public class CVPipeline2d extends CVPipeline { public CVPipeline2dResult(List targets, Mat outputMat) { - this.targets = targets; - this.hasTarget = !targets.isEmpty(); - this.outputMat = outputMat; + super(targets, outputMat); } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java new file mode 100644 index 000000000..43893215d --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline2dSettings.java @@ -0,0 +1,29 @@ +package com.chameleonvision.classabstraction.pipeline; + +import com.chameleonvision.vision.CalibrationMode; +import com.chameleonvision.vision.SortMode; +import com.chameleonvision.vision.TargetGroup; +import com.chameleonvision.vision.TargetIntersection; + +import java.util.Arrays; +import java.util.List; + +public class CVPipeline2dSettings extends CVPipelineSettings { + public List hue = Arrays.asList(50, 180); + public List saturation = Arrays.asList(50, 255); + public List value = Arrays.asList(50, 255); + public boolean erode = false; + public boolean dilate = false; + public List area = Arrays.asList(0.0, 100.0); + public List ratio = Arrays.asList(0.0, 20.0); + public List extent = Arrays.asList(0, 100); + public Number speckle = 5; + public boolean isBinary = false; + public SortMode sortMode = SortMode.Largest; + public TargetGroup targetGroup = TargetGroup.Single; + public TargetIntersection targetIntersection = TargetIntersection.Up; + public List point = Arrays.asList(0, 0); + public CalibrationMode calibrationMode = CalibrationMode.None; + public double dualTargetCalibrationM = 1; + public double dualTargetCalibrationB = 0; +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java index 9fa8a6bd1..66029cfe0 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3d.java @@ -1,33 +1,33 @@ -//package com.chameleonvision.classabstraction.pipeline; -// -//import org.opencv.core.Mat; -// -//public class CVPipeline3d extends CVPipeline { -// -// public CVPipeline3d(CVPipelineSettings settings) { -// super(settings); -// } -// -// @Override -// void initPipeline() { -// -// } -// -// @Override -// CVPipeline3d.CVPipeline3dResult runPipeline(Mat inputMat) { -// return null; -// } -// -// @Override -// Mat getOutputMat() { -// return null; -// } -// -// public static class CVPipeline3dSettings extends CVPipelineSettings { -// } -// -// public static class CVPipeline3dResult { -// -// } -// -//} +package com.chameleonvision.classabstraction.pipeline; + +import com.chameleonvision.classabstraction.camera.CameraProcess; +import org.opencv.core.Mat; + +import java.util.List; +import java.util.function.Supplier; + +import static com.chameleonvision.classabstraction.pipeline.CVPipeline3d.*; + +public class CVPipeline3d extends CVPipeline { + + public CVPipeline3d(Supplier settingsSupplier) { + super(settingsSupplier); + } + + @Override + public CVPipeline3dResult runPipeline(Mat inputMat) { + return null; + } + + + public static class CVPipeline3dResult extends CVPipelineResult { + public CVPipeline3dResult(List targets, Mat outputMat) { + super(targets, outputMat); + } + } + + public static class Target3d { + // TODO: Define 3d-specific target data + } + +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java new file mode 100644 index 000000000..e9a5be7ba --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipeline3dSettings.java @@ -0,0 +1,5 @@ +package com.chameleonvision.classabstraction.pipeline; + +public class CVPipeline3dSettings extends CVPipeline2dSettings { + // TODO: +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java index eb639ea53..3ffe26d9d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineResult.java @@ -5,7 +5,13 @@ import org.opencv.core.Mat; import java.util.List; public abstract class CVPipelineResult { - List targets; - boolean hasTarget; - Mat outputMat; + public final List targets; + public final boolean hasTarget; + public final Mat outputMat = new Mat(); + + public CVPipelineResult(List targets, Mat outputMat) { + this.targets = targets; + hasTarget = targets != null && !targets.isEmpty(); + outputMat.copyTo(this.outputMat); + } } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java index 79909b0fa..644461a9d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/CVPipelineSettings.java @@ -2,29 +2,10 @@ package com.chameleonvision.classabstraction.pipeline; import com.chameleonvision.vision.*; -import java.util.Arrays; -import java.util.List; - @SuppressWarnings("ALL") -public abstract class CVPipelineSettings { - ImageFlipMode flipMode = ImageFlipMode.NONE; - List hue = Arrays.asList(50, 180); - List saturation = Arrays.asList(50, 255); - List value = Arrays.asList(50, 255); - boolean erode = false; - boolean dilate = false; - List area = Arrays.asList(0.0, 100.0); - List ratio = Arrays.asList(0.0, 20.0); - List extent = Arrays.asList(0, 100); - Number speckle = 5; - boolean isBinary = false; - SortMode sortMode = SortMode.Largest; - TargetGroup targetGroup = TargetGroup.Single; - TargetIntersection targetIntersection = TargetIntersection.Up; - List point = Arrays.asList(0,0); - CalibrationMode calibrationMode = CalibrationMode.None; - - String nickname = ""; - double exposure = 50.0; - double brightness = 50.0; +public class CVPipelineSettings { + public ImageFlipMode flipMode = ImageFlipMode.NONE; + public String nickname = ""; + public double exposure = 50.0; + public double brightness = 50.0; } diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java index 82a8ee333..c59a5ea83 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/DriverVisionPipeline.java @@ -3,26 +3,34 @@ package com.chameleonvision.classabstraction.pipeline; import com.chameleonvision.classabstraction.camera.CameraProcess; import org.opencv.core.Mat; -public class DriverVisionPipeline extends CVPipeline { - public DriverVisionPipeline(CVPipelineSettings settings) { - super(settings); +import java.util.List; +import java.util.function.Supplier; + +import static com.chameleonvision.classabstraction.pipeline.DriverVisionPipeline.*; + +public class DriverVisionPipeline extends CVPipeline { + + public DriverVisionPipeline(Supplier settingsSupplier) { + super(settingsSupplier); } @Override - void initPipeline(CameraProcess camera) { - // TODO: set camera to driver mode + public void initPipeline(CameraProcess camera) { + camera.setBrightness((int) getSettings().brightness); + camera.setExposure((int) getSettings().exposure); } @Override - DriverPipelineResult runPipeline(Mat inputMat) { - return new DriverPipelineResult(inputMat); + public DriverPipelineResult runPipeline(Mat inputMat) { + + inputMat.copyTo(outputMat); + + return new DriverPipelineResult(null, inputMat); } public static class DriverPipelineResult extends CVPipelineResult { - public DriverPipelineResult(Mat outputMat) { - this.hasTarget = false; - this.targets = null; - outputMat.copyTo(this.outputMat); + public DriverPipelineResult(List targets, Mat outputMat) { + super(targets, outputMat); } } } diff --git a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java index 44caa86f9..30d460ce3 100644 --- a/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/process/VisionProcess.java @@ -1,5 +1,7 @@ package com.chameleonvision.vision.process; +import com.chameleonvision.classabstraction.pipeline.CVPipeline2d; +import com.chameleonvision.classabstraction.pipeline.DriverVisionPipeline; import com.chameleonvision.settings.SettingsManager; import com.chameleonvision.vision.Orientation; import com.chameleonvision.vision.Pipeline; @@ -12,6 +14,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import static com.chameleonvision.classabstraction.pipeline.CVPipeline2d.*; + public class VisionProcess implements Runnable { private final String cameraName; @@ -29,6 +33,10 @@ public class VisionProcess implements Runnable { // chameleon specific private Pipeline currentPipeline; private CVProcess cvProcess; + + private CVPipeline2d cvPipeline2d; + private DriverVisionPipeline driverVisionPipeline; + // pipeline process items // private List foundContours = new ArrayList<>(); // private List filteredContours = new ArrayList<>(); @@ -41,7 +49,6 @@ public class VisionProcess implements Runnable { public VisionProcess(CameraProcess cameraProcess) { - // USBCameraProcess settings cvProcess = new StandardCVProcess(cameraProcess.getCamVals()); this.cameraProcess = cameraProcess; // new USBCameraProcess(cameraProcess); @@ -95,15 +102,58 @@ public class VisionProcess implements Runnable { } } + private CVPipeline2dSettings pipelineTo2dSettings(Pipeline pipeline) { + CVPipeline2dSettings settings = new CVPipeline2dSettings(); + settings.hue = pipeline.hue; + settings.saturation = pipeline.saturation; + settings.value = pipeline.value; + settings.erode = pipeline.erode; + settings.dilate = pipeline.dilate; + settings.area = pipeline.area; + settings.ratio = pipeline.ratio; + settings.extent = pipeline.extent; + settings.speckle = pipeline.speckle; + settings.isBinary = pipeline.isBinary; + settings.sortMode = pipeline.sortMode; + settings.targetGroup = pipeline.targetGroup; + settings.targetIntersection = pipeline.targetIntersection; + settings.point = pipeline.point; + settings.calibrationMode = pipeline.calibrationMode; + settings.nickname = pipeline.nickname; + settings.exposure = pipeline.exposure; + settings.brightness = pipeline.brightness; + settings.dualTargetCalibrationM = pipeline.m; + settings.dualTargetCalibrationB = pipeline.b; + + return settings; + } + private PipelineResult runVisionProcess(Mat inputImage, Mat outputImage) { - return cvProcess.runPipeline( - currentPipeline, - inputImage, - outputImage, - cameraProcess.getCamVals(), - currentPipeline.orientation.equals(Orientation.Inverted), - cameraProcess.getDriverMode() - ); + + if (cvPipeline2d == null) { + cvPipeline2d = new CVPipeline2d(() -> pipelineTo2dSettings(currentPipeline)); + } + CVPipeline2dResult result = cvPipeline2d.runPipeline(inputImage); + result.outputMat.copyTo(outputImage); + + PipelineResult pipeResult = new PipelineResult(); + pipeResult.IsValid = result.hasTarget; + if (!pipeResult.IsValid) { + pipeResult.CalibratedX = 0; + pipeResult.CalibratedY = 0; + pipeResult.Pitch = 0; + pipeResult.Yaw = 0; + pipeResult.Area = 0; + } else { + Target t = result.targets.get(0); + pipeResult.CalibratedX = t.calibratedX; + pipeResult.CalibratedY = t.calibratedY; + pipeResult.Pitch = t.pitch; + pipeResult.Yaw = t.yaw; + pipeResult.Area = t.area; + } + + return pipeResult; } @Override From b1e8fb53a5415b379438aa28ecd3552388cbdef4 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 10 Nov 2019 17:55:28 -0500 Subject: [PATCH 13/84] Add PipelineType enum for config --- .../chameleonvision/classabstraction/VisionManager.java | 6 +++--- .../classabstraction/camera/USBCameraProcess.java | 1 + .../classabstraction/{camera => config}/CameraConfig.java | 2 +- .../classabstraction/config/PipelineConfig.java | 8 ++++++++ .../classabstraction/pipeline/PipelineType.java | 7 +++++++ 5 files changed, 20 insertions(+), 4 deletions(-) rename Main/src/main/java/com/chameleonvision/classabstraction/{camera => config}/CameraConfig.java (80%) create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/config/PipelineConfig.java create mode 100644 Main/src/main/java/com/chameleonvision/classabstraction/pipeline/PipelineType.java diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java b/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java index e5d735f2f..7e8ae11bc 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/VisionManager.java @@ -47,9 +47,9 @@ public class VisionManager { Path cameraConfigPath = Paths.get(CamConfigPath.toString(), String.format("%s.json", cameraName)); File cameraConfigFile = new File(cameraConfigPath.toString()); if (cameraConfigFile.exists() && cameraConfigFile.length() != 0) { - try { - Gson gson = new GsonBuilder()().registerTypeAdapter(USBCameraProcess.class, new CameraDeserializer()) - } +// try { +// Gson gson = new GsonBuilder().registerTypeAdapter(USBCameraProcess.class, new CameraDeserializer()); +// } } }) diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java index be5b506ef..f24b15e3d 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/camera/USBCameraProcess.java @@ -1,5 +1,6 @@ package com.chameleonvision.classabstraction.camera; +import com.chameleonvision.classabstraction.config.CameraConfig; import edu.wpi.cscore.CvSink; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.VideoException; diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java b/Main/src/main/java/com/chameleonvision/classabstraction/config/CameraConfig.java similarity index 80% rename from Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java rename to Main/src/main/java/com/chameleonvision/classabstraction/config/CameraConfig.java index 830b9a358..22944dd27 100644 --- a/Main/src/main/java/com/chameleonvision/classabstraction/camera/CameraConfig.java +++ b/Main/src/main/java/com/chameleonvision/classabstraction/config/CameraConfig.java @@ -1,4 +1,4 @@ -package com.chameleonvision.classabstraction.camera; +package com.chameleonvision.classabstraction.config; public class CameraConfig { public double FOV; diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/config/PipelineConfig.java b/Main/src/main/java/com/chameleonvision/classabstraction/config/PipelineConfig.java new file mode 100644 index 000000000..7fa50505f --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/config/PipelineConfig.java @@ -0,0 +1,8 @@ +package com.chameleonvision.classabstraction.config; + +import com.chameleonvision.classabstraction.pipeline.PipelineType; + +public class PipelineConfig { + public PipelineType type; + // TODO: config stuff +} diff --git a/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/PipelineType.java b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/PipelineType.java new file mode 100644 index 000000000..4b705fd36 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/classabstraction/pipeline/PipelineType.java @@ -0,0 +1,7 @@ +package com.chameleonvision.classabstraction.pipeline; + +public enum PipelineType { + PIPELINE_2D, + PIPELINE_3D, + PIPELINE_DRIVER; +} From 5bc2afe29ac8cbc727dea4f3b675137e15114d3c Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 11 Nov 2019 17:26:32 +0200 Subject: [PATCH 14/84] initial work on pipeline json --- Main/chameleon-vision.iml | 19 +++---------------- Main/pom.xml | 17 +++++++++++++++-- .../classabstraction/VisionManager.java | 19 +++++++++++++++++++ tets/src/main/java/com/animal.java | 4 ++++ tets/src/main/java/com/cat.java | 3 +++ tets/src/main/java/com/cow.java | 3 +++ tets/src/main/java/main.java | 4 ++++ 7 files changed, 51 insertions(+), 18 deletions(-) create mode 100644 tets/src/main/java/com/animal.java create mode 100644 tets/src/main/java/com/cat.java create mode 100644 tets/src/main/java/com/cow.java create mode 100644 tets/src/main/java/main.java diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 6f10ef5f5..487436a6c 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -12,19 +12,6 @@ - - - - - - - - - - - - - @@ -54,9 +41,9 @@ - - - + + + diff --git a/Main/pom.xml b/Main/pom.xml index a96fc2933..a383f5067 100644 --- a/Main/pom.xml +++ b/Main/pom.xml @@ -108,10 +108,23 @@ com.fasterxml.jackson.core - jackson-databind - 2.10.0.pr1 + jackson-core + 2.9.6 + + com.fasterxml.jackson.core + jackson-annotations + 2.9.6 + + + + com.fasterxml.jackson.core + jackson-databind + 2.9.6 + + + @@ -106,25 +106,25 @@ commons-lang3 3.9 + com.fasterxml.jackson.core jackson-core - 2.10.0 + 2.10.1 com.fasterxml.jackson.core jackson-annotations - 2.10.0 + 2.10.1 com.fasterxml.jackson.core jackson-databind - 2.10.0 + 2.10.1 - edu.wpi.first.cscore cscore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf linuxaarch64bionic edu.wpi.first.cscore cscore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf linuxraspbian edu.wpi.first.cscore cscore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf linuxx86-64 edu.wpi.first.cscore cscore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf osxx86-64 edu.wpi.first.cscore cscore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf windowsx86-64 From 159e628357717d9017d01742719d0ae5b173bb87 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 23 Nov 2019 20:39:04 -0800 Subject: [PATCH 55/84] Update chameleon-vision.iml --- Main/chameleon-vision.iml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index b843db04b..72eb49dc1 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -44,12 +44,12 @@ - - - - - - + + + + + + From 3852e6d908853b94a04e7ae2b1ecfdce287982ff Mon Sep 17 00:00:00 2001 From: ori agranat Date: Sun, 24 Nov 2019 23:51:12 +0200 Subject: [PATCH 56/84] removed unnecessary call for send full settings --- Main/src/main/java/com/chameleonvision/vision/VisionManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java index 71c3f9366..c6d69d194 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java @@ -99,7 +99,6 @@ public class VisionManager { VisionProcesses.forEach((vpm) -> { vpm.visionProcess.start(); }); - ServerHandler.sendFullSettings(); } public static VisionProcess getCurrentUIVisionProcess() { From 5841122091a73b1eafae9be07c8b17d5039aadc5 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 24 Nov 2019 19:29:37 -0500 Subject: [PATCH 57/84] Fix WPILib dep versions --- Main/pom.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Main/pom.xml b/Main/pom.xml index 9b17721e8..9dab6dc06 100644 --- a/Main/pom.xml +++ b/Main/pom.xml @@ -174,39 +174,39 @@ edu.wpi.first.cameraserver cameraserver-java - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf edu.wpi.first.ntcore ntcore-java - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf edu.wpi.first.ntcore ntcore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf osxx86-64 edu.wpi.first.ntcore ntcore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf linuxraspbian edu.wpi.first.ntcore ntcore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf linuxx86-64 edu.wpi.first.ntcore ntcore-jni - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf windowsx86-64 @@ -214,7 +214,7 @@ edu.wpi.first.wpiutil wpiutil-java - 2019.4.1-213-g56d782b + 2020.1.1-beta-2-98-gb058dcf From 80d4451e56809fb53eec0684464134d605fe63bd Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 24 Nov 2019 19:31:45 -0500 Subject: [PATCH 58/84] Fix timing calculations, add OpenCV exception catching to some pipes --- .../chameleonvision/vision/VisionProcess.java | 10 ++-- .../vision/pipeline/CVPipeline2d.java | 51 ++++++++++++------- .../vision/pipeline/pipes/BlurPipe.java | 19 ++++--- .../pipeline/pipes/Collect2dTargetsPipe.java | 2 +- .../pipeline/pipes/Draw2dContoursPipe.java | 2 +- .../pipeline/pipes/ErodeDilatePipe.java | 3 +- .../pipeline/pipes/FilterContoursPipe.java | 2 +- .../pipeline/pipes/FindContoursPipe.java | 2 +- .../pipeline/pipes/GroupContoursPipe.java | 2 +- .../vision/pipeline/pipes/HsvPipe.java | 15 +++--- .../vision/pipeline/pipes/OutputMatPipe.java | 2 +- .../vision/pipeline/pipes/RotateFlipPipe.java | 3 +- .../pipeline/pipes/SortContoursPipe.java | 2 +- .../pipeline/pipes/SpeckleRejectPipe.java | 2 +- 14 files changed, 64 insertions(+), 53 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index bc75347fa..578171845 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -362,13 +362,11 @@ public class VisionProcess { result = currentPipeline.runPipeline(streamBuffer); lastPipelineResult = result; - var yes = lastPipelineResult==null; + if (result != null) { + updateNetworkTableData(lastPipelineResult); + updateUI(lastPipelineResult); + } - updateNetworkTableData(lastPipelineResult); - updateUI(lastPipelineResult); - - } else { -// System.err.println("Bad streambuffer mat"); } var deltaTimeNanos = lastUpdateTimeNanos - System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index e9645a2f0..fd057df30 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -31,6 +31,8 @@ public class CVPipeline2d extends CVPipeline rotateFlipResult = rotateFlipPipe.run(inputMat); totalProcessTimeNanos += rotateFlipResult.getRight(); - procTimeStringBuilder.append(String.format("RotateFlip: %.2fms, ", rotateFlipResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("RotateFlip: %.2fms, ", rotateFlipResult.getRight() / 1000000.0)); Pair blurResult = blurPipe.run(rotateFlipResult.getLeft()); totalProcessTimeNanos += blurResult.getRight(); - procTimeStringBuilder.append(String.format("Blur: %.2fms, ", blurResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("Blur: %.2fms, ", blurResult.getRight() / 1000000.0)); Pair erodeDilateResult = erodeDilatePipe.run(blurResult.getLeft()); totalProcessTimeNanos += erodeDilateResult.getRight(); - procTimeStringBuilder.append(String.format("ErodeDilate: %.2fms, ", erodeDilateResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("ErodeDilate: %.2fms, ", erodeDilateResult.getRight() / 1000000.0)); Pair hsvResult = hsvPipe.run(erodeDilateResult.getLeft()); totalProcessTimeNanos += hsvResult.getRight(); - Imgproc.cvtColor(hsvResult.getLeft(), hsvOutputMat, Imgproc.COLOR_GRAY2BGR, 3); - procTimeStringBuilder.append(String.format("HSV: %.2fms, ", hsvResult.getRight() / 1000.0)); + + try { + Imgproc.cvtColor(hsvResult.getLeft(), hsvOutputMat, Imgproc.COLOR_GRAY2BGR, 3); + } catch (CvException e) { + System.err.println("(CVPipeline2d) Exception thrown by OpenCV: \n" + e.getMessage()); + } + + procTimeStringBuilder.append(String.format("HSV: %.2fms, ", hsvResult.getRight() / 1000000.0)); Pair, Long> findContoursResult = findContoursPipe.run(hsvResult.getLeft()); totalProcessTimeNanos += findContoursResult.getRight(); - procTimeStringBuilder.append(String.format("FindContours: %.2fms, ", findContoursResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("FindContours: %.2fms, ", findContoursResult.getRight() / 1000000.0)); Pair, Long> filterContoursResult = filterContoursPipe.run(findContoursResult.getLeft()); totalProcessTimeNanos += filterContoursResult.getRight(); - procTimeStringBuilder.append(String.format("FilterContours: %.2fms, ", filterContoursResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("FilterContours: %.2fms, ", filterContoursResult.getRight() / 1000000.0)); Pair, Long> speckleRejectResult = speckleRejectPipe.run(filterContoursResult.getLeft()); totalProcessTimeNanos += speckleRejectResult.getRight(); - procTimeStringBuilder.append(String.format("SpeckleReject: %.2fms, ", speckleRejectResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("SpeckleReject: %.2fms, ", speckleRejectResult.getRight() / 1000000.0)); Pair, Long> groupContoursResult = groupContoursPipe.run(speckleRejectResult.getLeft()); totalProcessTimeNanos += groupContoursResult.getRight(); - procTimeStringBuilder.append(String.format("GroupContours: %.2fms, ", groupContoursResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("GroupContours: %.2fms, ", groupContoursResult.getRight() / 1000000.0)); Pair, Long> sortContoursResult = sortContoursPipe.run(groupContoursResult.getLeft()); totalProcessTimeNanos += sortContoursResult.getRight(); - procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); Pair, Long> collect2dTargetsResult = collect2dTargetsPipe.run(sortContoursResult.getLeft()); totalProcessTimeNanos += collect2dTargetsResult.getRight(); - procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); // takes pair of (Mat of original camera image, Mat of HSV thresholded image) Pair outputMatResult = outputMatPipe.run(Pair.of(rawCameraMat, hsvOutputMat)); totalProcessTimeNanos += outputMatResult.getRight(); - procTimeStringBuilder.append(String.format("OutputMat: %.2fms, ", outputMatResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("OutputMat: %.2fms, ", outputMatResult.getRight() / 1000000.0)); // takes pair of (Mat to draw on, List of sorted contours) Pair draw2dContoursResult = draw2dContoursPipe.run(Pair.of(outputMatResult.getLeft(), sortContoursResult.getLeft())); totalProcessTimeNanos += draw2dContoursResult.getRight(); - procTimeStringBuilder.append(String.format("Draw2dContours: %.2fms, ", draw2dContoursResult.getRight() / 1000.0)); + procTimeStringBuilder.append(String.format("Draw2dContours: %.2fms, ", draw2dContoursResult.getRight() / 1000000.0)); System.out.println(procTimeStringBuilder.toString()); - System.out.printf("Pipeline ran in %.3fms\n", totalProcessTimeNanos / 1000.0); + System.out.printf("Pipeline ran in %.3fms\n", totalProcessTimeNanos / 1000000.0); - return new CVPipeline2dResult(collect2dTargetsResult.getLeft(), draw2dContoursResult.getLeft(), totalProcessTimeNanos / 1000); + return new CVPipeline2dResult(collect2dTargetsResult.getLeft(), draw2dContoursResult.getLeft(), totalProcessTimeNanos); } public static class CVPipeline2dResult extends CVPipelineResult { - public CVPipeline2dResult(List targets, Mat outputMat, long processTime) { - super(targets, outputMat, processTime); + public CVPipeline2dResult(List targets, Mat outputMat, long processTimeNanos) { + super(targets, outputMat, processTimeNanos); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java index f490c4f7f..51a216a8e 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java @@ -1,7 +1,10 @@ package com.chameleonvision.vision.pipeline.pipes; import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.CvException; import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; public class BlurPipe implements Pipe { @@ -18,15 +21,15 @@ public class BlurPipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - // TODO (HIGH) make this blur - input.copyTo(processBuffer); -// if (blurSize > 0) { -// Imgproc.blur(outputMat, outputMat, new Size(blurSize, blurSize)); -// } + try { + if (blurSize > 0) { + Imgproc.blur(outputMat, outputMat, new Size(blurSize, blurSize)); + } + } catch (CvException e) { + System.err.println("(BlurPipe) Exception thrown by OpenCV: \n" + e.getMessage()); + } - processBuffer.copyTo(outputMat); - - long processTime = processStartNanos - System.nanoTime(); + long processTime = System.nanoTime() - processStartNanos; Pair output = Pair.of(outputMat, processTime); processBuffer.release(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java index 3d9fcf0ba..cb0111c76 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java @@ -56,7 +56,7 @@ public class Collect2dTargetsPipe implements Pipe, List>, Ma } } - long processTime = processStartNanos - System.nanoTime(); + long processTime = System.nanoTime() - processStartNanos; Pair output = Pair.of(outputMat, processTime); outputMat.release(); return output; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java index e0e86641c..adb26805f 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java @@ -35,8 +35,7 @@ public class ErodeDilatePipe implements Pipe { input.copyTo(processBuffer); } - long processTime = processStartNanos - System.nanoTime(); - processBuffer.copyTo(outputMat); + long processTime = System.nanoTime() - processStartNanos; Pair output = Pair.of(outputMat, processTime); processBuffer.release(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java index 404bbcd5a..429b79052 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java @@ -56,7 +56,7 @@ public class FilterContoursPipe implements Pipe, List> { Imgproc.findContours(input, foundContours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); - long processTime = processStartNanos - System.nanoTime(); + long processTime = System.nanoTime() - processStartNanos; return Pair.of(foundContours, processTime); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java index db3a374e6..4f9581b31 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java @@ -85,7 +85,7 @@ public class GroupContoursPipe implements Pipe, List { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - // convert from rgb to hsv - if(input.empty()) { - throw new RuntimeException("HSV input cannot be empty!"); + try { + Imgproc.cvtColor(input, outputMat, Imgproc.COLOR_RGB2HSV, 3); + Core.inRange(outputMat, hsvLower, hsvUpper, outputMat); + } catch (CvException e) { + System.err.println("(HsvPipe) Exception thrown by OpenCV: \n" + e.getMessage()); } - Imgproc.cvtColor(input, processBuffer, Imgproc.COLOR_RGB2HSV, 3); - Core.inRange(processBuffer, hsvLower, hsvUpper, processBuffer); - - long processTime = processStartNanos - System.nanoTime(); - processBuffer.copyTo(outputMat); + long processTime = System.nanoTime() - processStartNanos; Pair output = Pair.of(outputMat, processTime); processBuffer.release(); return output; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java index f0dc2af3d..7593a2ca8 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java @@ -19,7 +19,7 @@ public class OutputMatPipe implements Pipe, Mat> { outputMat = showThresholded ? input.getRight() : input.getLeft(); - long processTime = processStartNanos - System.nanoTime(); + long processTime = System.nanoTime() - processStartNanos; Pair output = Pair.of(outputMat, processTime); outputMat.release(); return output; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java index 03a67ccdd..cb7378d5c 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java @@ -26,8 +26,7 @@ public class RotateFlipPipe implements Pipe { Core.flip(input, processBuffer, flip.value); Core.rotate(processBuffer, processBuffer, rotation.value); - long processTime = processStartNanos - System.nanoTime(); - processBuffer.copyTo(outputMat); + long processTime = System.nanoTime() - processStartNanos; Pair output = Pair.of(outputMat, processTime); processBuffer.release(); return output; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java index 1a6934849..89454222a 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java @@ -64,7 +64,7 @@ public class SortContoursPipe implements Pipe, List, List Date: Sun, 24 Nov 2019 16:54:10 -0800 Subject: [PATCH 59/84] stop instantiating new pipes every tick --- Main/chameleon-vision.iml | 20 ++++--- .../vision/pipeline/CVPipeline2d.java | 52 +++++++++++++++---- .../vision/pipeline/pipes/BlurPipe.java | 6 ++- .../pipeline/pipes/Collect2dTargetsPipe.java | 20 +++++-- .../pipeline/pipes/ErodeDilatePipe.java | 11 +++- .../pipeline/pipes/FilterContoursPipe.java | 13 ++++- .../pipeline/pipes/GroupContoursPipe.java | 9 +++- .../vision/pipeline/pipes/HsvPipe.java | 8 ++- .../vision/pipeline/pipes/RotateFlipPipe.java | 9 +++- .../pipeline/pipes/SortContoursPipe.java | 9 +++- .../pipeline/pipes/SpeckleRejectPipe.java | 6 ++- 11 files changed, 127 insertions(+), 36 deletions(-) diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 72eb49dc1..6f3e66cfc 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -12,6 +12,12 @@ + + + + + + @@ -50,13 +56,13 @@ - - - - - - - + + + + + + + diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index fd057df30..34d99037b 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -1,5 +1,6 @@ package com.chameleonvision.vision.pipeline; +import com.chameleonvision.vision.camera.CameraProcess; import com.chameleonvision.vision.camera.CameraStaticProperties; import com.chameleonvision.vision.pipeline.pipes.*; import com.chameleonvision.vision.enums.ImageRotation; @@ -16,6 +17,16 @@ public class CVPipeline2d extends CVPipeline { - private final int blurSize; + private int blurSize; private Mat processBuffer = new Mat(); private Mat outputMat = new Mat(); @@ -17,6 +17,10 @@ public class BlurPipe implements Pipe { this.blurSize = blurSize; } + public void setConfig(int blurSize) { + this.blurSize = blurSize; + } + @Override public Pair run(Mat input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java index cb0111c76..c035e138e 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java @@ -12,14 +12,24 @@ import java.util.List; public class Collect2dTargetsPipe implements Pipe, List> { - private final CalibrationMode calibrationMode; - private final CameraStaticProperties camProps; - private final List calibrationPoint; - private final double calibrationM, calibrationB; + private CalibrationMode calibrationMode; + private CameraStaticProperties camProps; + private List calibrationPoint; + private double calibrationM, calibrationB; private List targets = new ArrayList<>(); - public Collect2dTargetsPipe(CalibrationMode calibrationMode, List calibrationPoint, double calibrationM, double calibrationB, CameraStaticProperties camProps) { + public Collect2dTargetsPipe(CalibrationMode calibrationMode, List calibrationPoint, + double calibrationM, double calibrationB, CameraStaticProperties camProps) { + this.calibrationMode = calibrationMode; + this.camProps = camProps; + this.calibrationPoint = calibrationPoint; + this.calibrationM = calibrationM; + this.calibrationB = calibrationB; + } + + public void setConfig(CalibrationMode calibrationMode, List calibrationPoint, + double calibrationM, double calibrationB, CameraStaticProperties camProps) { this.calibrationMode = calibrationMode; this.camProps = camProps; this.calibrationPoint = calibrationPoint; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java index adb26805f..3ecbef9c4 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java @@ -7,8 +7,9 @@ import org.opencv.imgproc.Imgproc; public class ErodeDilatePipe implements Pipe { - private final boolean erode, dilate; - private final Mat kernel; + private boolean erode; + private boolean dilate; + private Mat kernel; private Mat processBuffer = new Mat(); private Mat outputMat = new Mat(); @@ -19,6 +20,12 @@ public class ErodeDilatePipe implements Pipe { kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(kernelSize, kernelSize)); } + public void setConfig(boolean erode, boolean dilate, int kernelSize) { + this.erode = erode; + this.dilate = dilate; + kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(kernelSize, kernelSize)); + } + @Override public Pair run(Mat input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java index 429b79052..b77b04016 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java @@ -13,8 +13,10 @@ import java.util.List; public class FilterContoursPipe implements Pipe, List> { - private final List area, ratio, extent; - private final CameraStaticProperties camProps; + private List area; + private List ratio; + private List extent; + private CameraStaticProperties camProps; private List filteredContours = new ArrayList<>(); @@ -25,6 +27,13 @@ public class FilterContoursPipe implements Pipe, List area, List ratio, List extent, CameraStaticProperties camProps) { + this.area = area; + this.ratio = ratio; + this.extent = extent; + this.camProps = camProps; + } + @Override public Pair, Long> run(List input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java index 4f9581b31..89bc37cc3 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java @@ -18,8 +18,8 @@ public class GroupContoursPipe implements Pipe, List sortByMomentsX = Comparator.comparingDouble(GroupContoursPipe::calcMomentsX); - private final TargetGroup group; - private final TargetIntersection intersection; + private TargetGroup group; + private TargetIntersection intersection; private List groupedContours = new ArrayList<>(); private MatOfPoint2f intersectMatA = new MatOfPoint2f(); @@ -30,6 +30,11 @@ public class GroupContoursPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java index 9fd5fe42e..945a07ab1 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java @@ -9,7 +9,8 @@ import org.opencv.imgproc.Imgproc; public class HsvPipe implements Pipe { - private final Scalar hsvLower, hsvUpper; + private Scalar hsvLower; + private Scalar hsvUpper; private Mat processBuffer = new Mat(); private Mat outputMat = new Mat(); @@ -19,6 +20,11 @@ public class HsvPipe implements Pipe { this.hsvUpper = hsvUpper; } + public void setConfig(Scalar hsvLower, Scalar hsvUpper) { + this.hsvLower = hsvLower; + this.hsvUpper = hsvUpper; + } + @Override public Pair run(Mat input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java index cb7378d5c..674b4ee16 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java @@ -8,8 +8,8 @@ import org.opencv.core.Mat; public class RotateFlipPipe implements Pipe { - private final ImageRotation rotation; - private final ImageFlipMode flip; + private ImageRotation rotation; + private ImageFlipMode flip; private Mat processBuffer = new Mat(); private Mat outputMat = new Mat(); @@ -19,6 +19,11 @@ public class RotateFlipPipe implements Pipe { this.flip = flip; } + public void setConfig(ImageRotation rotation, ImageFlipMode flip) { + this.rotation = rotation; + this.flip = flip; + } + @Override public Pair run(Mat input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java index 89454222a..457a7ae68 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java @@ -24,8 +24,8 @@ public class SortContoursPipe implements Pipe, List SortByRightmostComparator = SortByLeftmostComparator.reversed(); - private final SortMode sort; - private final CameraStaticProperties camProps; + private SortMode sort; + private CameraStaticProperties camProps; private List sortedContours = new ArrayList<>(); @@ -34,6 +34,11 @@ public class SortContoursPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java index ebbd8ab52..5fe11e479 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java @@ -9,7 +9,7 @@ import java.util.List; public class SpeckleRejectPipe implements Pipe, List> { - private final double minPercentOfAvg; + private double minPercentOfAvg; private List despeckledContours = new ArrayList<>(); @@ -17,6 +17,10 @@ public class SpeckleRejectPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); From bb48b5a4dca2c9cbbd1a49d9d4451a24e7099a88 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 24 Nov 2019 19:34:35 -0500 Subject: [PATCH 60/84] Update chameleon-vision.iml --- Main/chameleon-vision.iml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 6f3e66cfc..d7ed36ac4 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -70,4 +70,4 @@ - \ No newline at end of file + From 88cfbc820a48f806e3b919ee88374399f18ed58f Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 24 Nov 2019 21:06:09 -0500 Subject: [PATCH 61/84] ConfigManager cleanups --- .../com/chameleonvision/config/ConfigManager.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java index 96dac65d7..dc5b30660 100644 --- a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java +++ b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java @@ -103,22 +103,27 @@ public class ConfigManager { public static List initializeCameraConfig(List preliminaryConfigs) { var configList = new ArrayList(); + checkSettingsFolder(); + // loop over all the camera names and try to create settings folders for it preliminaryConfigs.forEach((preliminaryConfig) -> { + String cameraName = preliminaryConfig.name; - final Path cameraConfigFolderPath = Paths.get(cameraConfigPath.toString(), String.format("%s\\", preliminaryConfig.name)); - final Path cameraConfigPath = Paths.get(cameraConfigFolderPath.toString(), "camera.json"); + final Path cameraConfigFolderPath = getCameraSpecificFolderPath(cameraName); + final Path cameraConfigPath = getCameraSpecificConfigPath(cameraName); // check if the config folder exists, and if not, create it - if (Files.notExists(cameraConfigFolderPath)) { + if (!cameraFolderExists(cameraName)) { try { Files.createDirectory(cameraConfigFolderPath); } catch (IOException e) { - e.printStackTrace(); + System.err.println("Failed to create camera config folder!"); } } else { CameraConfig config = preliminaryConfig; - if(!Files.exists(cameraConfigPath)) { + + // check if the config exists, and if not, create it + if(!cameraConfigExists(cameraName)) { try { FileHelper.Serializer(cameraConfigPath, preliminaryConfig); } catch (IOException e) { From 4975e5cfe8265a04c0331d629fda041cd94329bf Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Sun, 24 Nov 2019 21:07:22 -0500 Subject: [PATCH 62/84] Fix CvExceptions for pipes, CVPipeline2d cleanups --- .../chameleonvision/vision/pipeline/CVPipeline2d.java | 11 ++++------- .../vision/pipeline/pipes/BlurPipe.java | 5 ++++- .../vision/pipeline/pipes/ErodeDilatePipe.java | 10 ++++------ .../vision/pipeline/pipes/HsvPipe.java | 7 +++++-- .../vision/pipeline/pipes/RotateFlipPipe.java | 5 ++++- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index 34d99037b..c19410250 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -44,9 +44,9 @@ public class CVPipeline2d extends CVPipeline { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); + input.copyTo(processBuffer); + try { if (blurSize > 0) { - Imgproc.blur(outputMat, outputMat, new Size(blurSize, blurSize)); + Imgproc.blur(processBuffer, processBuffer, new Size(blurSize, blurSize)); } } catch (CvException e) { System.err.println("(BlurPipe) Exception thrown by OpenCV: \n" + e.getMessage()); } long processTime = System.nanoTime() - processStartNanos; + processBuffer.copyTo(outputMat); Pair output = Pair.of(outputMat, processTime); processBuffer.release(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java index 3ecbef9c4..4534f8ae1 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java @@ -30,21 +30,19 @@ public class ErodeDilatePipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); + input.copyTo(processBuffer); + if (erode) { - Imgproc.erode(input, processBuffer, kernel); + Imgproc.erode(processBuffer, processBuffer, kernel); } if (dilate) { Imgproc.erode(processBuffer, processBuffer, kernel); } - if(!erode && !dilate) { - input.copyTo(processBuffer); - } - long processTime = System.nanoTime() - processStartNanos; + processBuffer.copyTo(outputMat); Pair output = Pair.of(outputMat, processTime); - processBuffer.release(); return output; } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java index 945a07ab1..454f851d5 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java @@ -29,14 +29,17 @@ public class HsvPipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); + input.copyTo(processBuffer); + try { - Imgproc.cvtColor(input, outputMat, Imgproc.COLOR_RGB2HSV, 3); - Core.inRange(outputMat, hsvLower, hsvUpper, outputMat); + Imgproc.cvtColor(processBuffer, processBuffer, Imgproc.COLOR_RGB2HSV, 3); + Core.inRange(processBuffer, hsvLower, hsvUpper, processBuffer); } catch (CvException e) { System.err.println("(HsvPipe) Exception thrown by OpenCV: \n" + e.getMessage()); } long processTime = System.nanoTime() - processStartNanos; + processBuffer.copyTo(outputMat); Pair output = Pair.of(outputMat, processTime); processBuffer.release(); return output; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java index 674b4ee16..a4a604dfb 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java @@ -28,10 +28,13 @@ public class RotateFlipPipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - Core.flip(input, processBuffer, flip.value); + input.copyTo(processBuffer); + + Core.flip(processBuffer, processBuffer, flip.value); Core.rotate(processBuffer, processBuffer, rotation.value); long processTime = System.nanoTime() - processStartNanos; + processBuffer.copyTo(outputMat); Pair output = Pair.of(outputMat, processTime); processBuffer.release(); return output; From 0a1046ff3904b0bb24b862bc14f0213ff93ba901 Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 24 Nov 2019 18:45:53 -0800 Subject: [PATCH 63/84] Clear lists in Pipes --- .../pipeline/pipes/Collect2dTargetsPipe.java | 2 ++ .../pipeline/pipes/FilterContoursPipe.java | 2 ++ .../vision/pipeline/pipes/FindContoursPipe.java | 2 ++ .../pipeline/pipes/GroupContoursPipe.java | 2 ++ .../vision/pipeline/pipes/SortContoursPipe.java | 17 ++++++++++------- .../pipeline/pipes/SpeckleRejectPipe.java | 2 ++ 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java index c035e138e..51747cfdd 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java @@ -41,6 +41,8 @@ public class Collect2dTargetsPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); + targets.clear(); + input.forEach(r -> { CVPipeline2d.Target2d t = new CVPipeline2d.Target2d(); t.rawPoint = r; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java index b77b04016..97acea6aa 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java @@ -38,6 +38,8 @@ public class FilterContoursPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); + filteredContours.clear(); + for (MatOfPoint Contour : input) { try { double contourArea = Imgproc.contourArea(Contour); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java index 5a7f77adf..bf57d57d1 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java @@ -18,6 +18,8 @@ public class FindContoursPipe implements Pipe> { public Pair, Long> run(Mat input) { long processStartNanos = System.nanoTime(); + foundContours.clear(); + Imgproc.findContours(input, foundContours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_TC89_L1); long processTime = System.nanoTime() - processStartNanos; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java index 89bc37cc3..8ced26acf 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java @@ -39,6 +39,8 @@ public class GroupContoursPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); + groupedContours.clear(); + List sorted = new ArrayList<>(input); sorted.sort(sortByMomentsX); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java index 457a7ae68..a6e28f1c7 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java @@ -43,27 +43,30 @@ public class SortContoursPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); + sortedContours.clear(); + sortedContours.addAll(input); + switch (sort) { case Largest: - input.sort(SortByLargestComparator); + sortedContours.sort(SortByLargestComparator); break; case Smallest: - input.sort(SortBySmallestComparator); + sortedContours.sort(SortBySmallestComparator); break; case Highest: - input.sort(SortByHighestComparator); + sortedContours.sort(SortByHighestComparator); break; case Lowest: - input.sort(SortByLowestComparator); + sortedContours.sort(SortByLowestComparator); break; case Leftmost: - input.sort(SortByLeftmostComparator); + sortedContours.sort(SortByLeftmostComparator); break; case Rightmost: - input.sort(SortByRightmostComparator); + sortedContours.sort(SortByRightmostComparator); break; case Centermost: - input.sort(SortByCentermostComparator); + sortedContours.sort(SortByCentermostComparator); break; default: break; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java index 5fe11e479..7edebf7f7 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java @@ -25,6 +25,8 @@ public class SpeckleRejectPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); + despeckledContours.clear(); + double averageArea = 0.0; for (MatOfPoint c : input) { From 0ce00ebdd4a9cf645f9512aa365468f8c0d1d9ac Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 24 Nov 2019 20:46:25 -0800 Subject: [PATCH 64/84] send more settings to the UI --- .gitignore | 1 + Main/.gitignore | 5 +++ Main/chameleon-vision.iml | 21 ++++++++++- .../chameleonvision/vision/VisionManager.java | 37 +++++++++---------- .../chameleonvision/web/ServerHandler.java | 16 ++++++++ 5 files changed, 60 insertions(+), 20 deletions(-) create mode 100644 Main/.gitignore diff --git a/.gitignore b/.gitignore index e1a08cb49..badf4ec29 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,4 @@ Main/target New client/chameleon-client/node_modules/ Main/dependency-reduced-pom.xml Main/src/main/java/META-INF +*.prefs diff --git a/Main/.gitignore b/Main/.gitignore new file mode 100644 index 000000000..31636b185 --- /dev/null +++ b/Main/.gitignore @@ -0,0 +1,5 @@ +bin/* +.settings/* +.project +.classpath +*.prefs \ No newline at end of file diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index d7ed36ac4..7ef49e681 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -18,6 +18,25 @@ + + + + + + + + + + + + + + + + + + + @@ -70,4 +89,4 @@ - + \ No newline at end of file diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java index c6d69d194..b74708265 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java @@ -7,7 +7,6 @@ import com.chameleonvision.util.Platform; import com.chameleonvision.vision.camera.CameraProcess; import com.chameleonvision.vision.camera.USBCameraProcess; import com.chameleonvision.vision.pipeline.CVPipelineSettings; -import com.chameleonvision.web.ServerHandler; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.UsbCameraInfo; import org.opencv.videoio.VideoCapture; @@ -20,9 +19,9 @@ public class VisionManager { private VisionManager() { } - private static final LinkedHashMap UsbCameraInfosByCameraName = new LinkedHashMap<>(); - private static final LinkedList LoadedCameraConfigs = new LinkedList<>(); - private static final LinkedList VisionProcesses = new LinkedList<>(); + private static final LinkedHashMap usbCameraInfosByCameraName = new LinkedHashMap<>(); + private static final LinkedList loadedCameraConfigs = new LinkedList<>(); + private static final LinkedList visionProcesses = new LinkedList<>(); private static class VisionProcessManageable { public final int index; @@ -45,22 +44,22 @@ public class VisionManager { if (cap.isOpened()) { cap.release(); String name = info.name; - while (UsbCameraInfosByCameraName.containsKey(name)) { + while (usbCameraInfosByCameraName.containsKey(name)) { suffix++; name = String.format("%s (%d)", name, suffix); } - UsbCameraInfosByCameraName.put(name, info); + usbCameraInfosByCameraName.put(name, info); } } - if (UsbCameraInfosByCameraName.isEmpty()) { + if (usbCameraInfosByCameraName.isEmpty()) { return false; } // load the config List preliminaryConfigs = new ArrayList<>(); - UsbCameraInfosByCameraName.values().forEach((cameraInfo) -> { + usbCameraInfosByCameraName.values().forEach((cameraInfo) -> { String truePath; if (Platform.CurrentPlatform.isWindows()) { @@ -72,7 +71,7 @@ public class VisionManager { preliminaryConfigs.add(new CameraConfig(truePath, cameraInfo.name)); }); - LoadedCameraConfigs.addAll(ConfigManager.initializeCameraConfig(preliminaryConfigs)); + loadedCameraConfigs.addAll(ConfigManager.initializeCameraConfig(preliminaryConfigs)); // TODO: (HIGH) Load pipelines from json // UsbCameraInfosByCameraName.forEach((cameraName, cameraInfo) -> { @@ -85,18 +84,18 @@ public class VisionManager { } public static boolean initializeProcesses() { - for (int i = 0; i < LoadedCameraConfigs.size(); i++) { - CameraConfig config = LoadedCameraConfigs.get(i); + for (int i = 0; i < loadedCameraConfigs.size(); i++) { + CameraConfig config = loadedCameraConfigs.get(i); CameraProcess camera = new USBCameraProcess(config); VisionProcess process = new VisionProcess(camera, config.name); - VisionProcesses.add(new VisionProcessManageable(i, config.name, process)); + visionProcesses.add(new VisionProcessManageable(i, config.name, process)); } currentUIVisionProcess = getVisionProcessByIndex(0); return true; } public static void startProcesses() { - VisionProcesses.forEach((vpm) -> { + visionProcesses.forEach((vpm) -> { vpm.visionProcess.start(); }); } @@ -106,7 +105,7 @@ public class VisionManager { } public static void setCurrentProcessByIndex(int processIndex) { - if (processIndex > VisionProcesses.size() - 1) { + if (processIndex > visionProcesses.size() - 1) { return; } @@ -114,16 +113,16 @@ public class VisionManager { } public static VisionProcess getVisionProcessByIndex(int processIndex) { - if (processIndex > VisionProcesses.size() - 1) { + if (processIndex > visionProcesses.size() - 1) { return null; } - VisionProcessManageable vpm = VisionProcesses.stream().filter(manageable -> manageable.index == processIndex).findFirst().orElse(null); + VisionProcessManageable vpm = visionProcesses.stream().filter(manageable -> manageable.index == processIndex).findFirst().orElse(null); return vpm != null ? vpm.visionProcess : null; } public static List getAllCameraNicknames() { - return VisionProcesses.stream().map(vpm -> vpm.visionProcess.getCamera() + return visionProcesses.stream().map(vpm -> vpm.visionProcess.getCamera() .getProperties().getNickname()).collect(Collectors.toList()); } @@ -132,7 +131,7 @@ public class VisionManager { } public static void saveCameras() { - VisionProcesses.forEach((vpm) -> { + visionProcesses.forEach((vpm) -> { VisionProcess process = vpm.visionProcess; String cameraName = process.getCamera().getProperties().name; List pipelines = process.getPipelines().stream().map(cvPipeline -> cvPipeline.settings).collect(Collectors.toList()); @@ -153,7 +152,7 @@ public class VisionManager { } public static int getCurrentUIVisionProcessIndex() { - VisionProcessManageable vpm = VisionProcesses.stream().filter(v -> v.visionProcess == currentUIVisionProcess).findFirst().orElse(null); + VisionProcessManageable vpm = visionProcesses.stream().filter(v -> v.visionProcess == currentUIVisionProcess).findFirst().orElse(null); return vpm != null ? vpm.index : -1; } } diff --git a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java index 1160f9669..e453689c3 100644 --- a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java +++ b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java @@ -269,6 +269,19 @@ public class ServerHandler { return tmp; } + private static Map allFieldsToMap(Object obj) { + Map map = new HashMap<>(); + try { + Field[] fields = obj.getClass().getFields(); + for (Field field : fields) { + map.put(field.getName(), field.get(obj)); + } + } catch (IllegalAccessException e) { + System.err.println("Illegal Access error:" + Arrays.toString(e.getStackTrace())); + } + return map; + } + public static void sendFullSettings() { //General settings Map fullSettings = new HashMap<>(); @@ -278,6 +291,8 @@ public class ServerHandler { CVPipeline currentPipeline = currentProcess.getCurrentPipeline(); try { + fullSettings.putAll(allFieldsToMap(ConfigManager.settings)); + fullSettings.putAll(allFieldsToMap(currentPipeline)); fullSettings.put("settings", getOrdinalSettings()); fullSettings.put("cameraSettings", getOrdinalCameraSettings()); fullSettings.put("cameraList", VisionManager.getAllCameraNicknames()); @@ -286,6 +301,7 @@ public class ServerHandler { // TODO (HIGH) all of these settings! fullSettings.put("pipelineList", VisionManager.getCurrentCameraPipelineNicknames()); fullSettings.put("resolutionList", VisionManager.getCurrentCameraResolutionList()); + fullSettings.put("FOV", currentCamera.getProperties().FOV); fullSettings.put("port", currentProcess.cameraStreamer.getStreamPort()); fullSettings.put("currentPipelineIndex", VisionManager.getCurrentUIVisionProcess().getCurrentPipelineIndex()); fullSettings.put("currentCameraIndex", VisionManager.getCurrentUIVisionProcessIndex()); From 2ae65822918ac7bdcf28ec54c822d4a9a7a9c984 Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 24 Nov 2019 20:57:48 -0800 Subject: [PATCH 65/84] Instantiate auxListEntry --- .../src/main/java/com/chameleonvision/vision/VisionProcess.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index 578171845..1d941ed01 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -115,6 +115,7 @@ public class VisionProcess { ntAreaEntry = newTable.getEntry("area"); ntTimeStampEntry = newTable.getEntry("timestamp"); ntValidEntry = newTable.getEntry("is_valid"); + ntAuxListEntry = newTable.getEntry("aux_targets"); ntDriveModeListenerID = ntDriverModeEntry.addListener(this::setDriverMode, EntryListenerFlags.kUpdate); ntPipelineListenerID = ntPipelineEntry.addListener(this::setPipeline, EntryListenerFlags.kUpdate); ntDriverModeEntry.setBoolean(false); @@ -408,6 +409,7 @@ public class VisionProcess { if (latestMat != null && latestMat.cols() > 0) { latestMat.copyTo(streamBuffer); streamer.runStream(streamBuffer); + streamBuffer.release(); // if (toStreamMat != null && toStreamMat.cols() > 0) { // } else { // System.out.println("fuuuuck"); From ef60e1334d20964f26d2c00a26c1cbed57f41584 Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 24 Nov 2019 21:26:59 -0800 Subject: [PATCH 66/84] start on ui reintegration --- .../chameleonvision/vision/VisionProcess.java | 2 +- .../chameleonvision/web/ServerHandler.java | 51 +++++++++++++++---- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index 1d941ed01..3cb5c8ad1 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -214,10 +214,10 @@ public class VisionProcess { ntValidEntry.setBoolean(data.hasTarget); if(data.hasTarget && !(data instanceof DriverVisionPipeline.DriverPipelineResult)) { if(data instanceof CVPipeline2d.CVPipeline2dResult) { - ntTimeStampEntry.setDouble(data.processTime); //noinspection unchecked List targets = (List) data.targets; + ntTimeStampEntry.setDouble(data.processTime); ntPitchEntry.setDouble(targets.get(0).pitch); ntYawEntry.setDouble(targets.get(0).yaw); ntAreaEntry.setDouble(targets.get(0).area); diff --git a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java index e453689c3..35006e66e 100644 --- a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java +++ b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java @@ -1,10 +1,13 @@ package com.chameleonvision.web; +import com.chameleonvision.config.GeneralSettings; import com.chameleonvision.vision.VisionManager; import com.chameleonvision.vision.VisionProcess; import com.chameleonvision.vision.camera.CameraProcess; import com.chameleonvision.config.ConfigManager; +import com.chameleonvision.vision.enums.CalibrationMode; import com.chameleonvision.vision.pipeline.CVPipeline; +import com.chameleonvision.vision.pipeline.CVPipeline2dSettings; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import com.chameleonvision.vision.enums.StreamDivisor; import com.fasterxml.jackson.core.JsonProcessingException; @@ -269,19 +272,45 @@ public class ServerHandler { return tmp; } - private static Map allFieldsToMap(Object obj) { + private static Map settingsToMap(GeneralSettings settings) { Map map = new HashMap<>(); - try { - Field[] fields = obj.getClass().getFields(); - for (Field field : fields) { - map.put(field.getName(), field.get(obj)); - } - } catch (IllegalAccessException e) { - System.err.println("Illegal Access error:" + Arrays.toString(e.getStackTrace())); - } + map.put("team_number", settings.teamNumber); + map.put("connection_type", settings.connectionType); + map.put("ip", settings.ip); + map.put("gateway", settings.gateway); + map.put("netmask", settings.netmask); + map.put("hostname", settings.hostname); + map.put("curr_camera", settings.currentCamera); + map.put("curr_pipeline", settings.currentPipeline); + return map; } + private static Map pipelineToMap(CVPipelineSettings s) { + Map map = new HashMap<>(); + map.put("exposure", s.exposure); + map.put("brightness", s.brightness); + if(s instanceof CVPipeline2dSettings) { + var s_ = (CVPipeline2dSettings) s; + map.put("orientation", s.flipMode.name()); + map.put("hue", s_.hue); + map.put("saturation", s_.saturation); + map.put("value", s_.value); + map.put("erode", s_.erode); + map.put("dilate", s_.dilate); + map.put("area", s_.area); + map.put("ratio", s_.ratio); + map.put("extent", s_.extent); + map.put("is_binary", s_.isBinary); + map.put("sort_mode", s_.sortMode.name()); + map.put("target_group", s_.targetGroup.name()); + map.put("target_intersection", s_.targetIntersection.name()); + map.put("M", s_.dualTargetCalibrationM); + map.put("B", s_.dualTargetCalibrationB); + map.put("is_calibrated", !s_.calibrationMode.equals(CalibrationMode.None)); + } + return map; + } public static void sendFullSettings() { //General settings Map fullSettings = new HashMap<>(); @@ -291,8 +320,8 @@ public class ServerHandler { CVPipeline currentPipeline = currentProcess.getCurrentPipeline(); try { - fullSettings.putAll(allFieldsToMap(ConfigManager.settings)); - fullSettings.putAll(allFieldsToMap(currentPipeline)); + fullSettings.putAll(settingsToMap(ConfigManager.settings)); + fullSettings.putAll(pipelineToMap(currentPipeline.settings)); fullSettings.put("settings", getOrdinalSettings()); fullSettings.put("cameraSettings", getOrdinalCameraSettings()); fullSettings.put("cameraList", VisionManager.getAllCameraNicknames()); From 26e0bb733ba686c58a66bab5904f97f856f0ccf8 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Mon, 25 Nov 2019 00:32:57 -0500 Subject: [PATCH 67/84] Small cleanups, possible leak fix --- Main/src/main/java/com/chameleonvision/Main.java | 2 +- .../com/chameleonvision/vision/pipeline/CVPipeline2d.java | 2 +- .../vision/pipeline/pipes/GroupContoursPipe.java | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/Main.java b/Main/src/main/java/com/chameleonvision/Main.java index 03b23926a..ff32ba8d0 100644 --- a/Main/src/main/java/com/chameleonvision/Main.java +++ b/Main/src/main/java/com/chameleonvision/Main.java @@ -30,7 +30,7 @@ public class Main { private static boolean manageNetwork = true; private static boolean ignoreRoot = false; private static String ntClientModeServer = null; - private static boolean testMode = true; + private static boolean testMode = false; private static class NTLogger implements Consumer { diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index c19410250..d7e0144ab 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -173,7 +173,7 @@ public class CVPipeline2d extends CVPipeline { + public static class CVPipeline2dResult extends CVPipelineResult { public CVPipeline2dResult(List targets, Mat outputMat, long processTimeNanos) { super(targets, outputMat, processTimeNanos); } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java index 8ced26acf..070a895a8 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java @@ -72,7 +72,9 @@ public class GroupContoursPipe implements Pipe, List Date: Mon, 25 Nov 2019 05:34:04 -0500 Subject: [PATCH 68/84] Further abstract image capture, fix some UI bugs --- Main/chameleon-vision.iml | 25 ++++----------- .../main/java/com/chameleonvision/Main.java | 2 +- .../chameleonvision/config/CameraConfig.java | 4 +-- .../chameleonvision/config/ConfigManager.java | 4 ++- .../chameleonvision/vision/VisionManager.java | 8 ++--- .../chameleonvision/vision/VisionProcess.java | 29 ++++++++--------- ...{CameraProcess.java => CameraCapture.java} | 11 ++----- .../vision/camera/CameraStreamer.java | 20 ++++++------ ...ties.java => CaptureStaticProperties.java} | 4 +-- ...meraProcess.java => USBCameraCapture.java} | 4 +-- .../vision/camera/USBCameraProperties.java | 22 ++++--------- .../vision/image/CaptureProperties.java | 20 ++++++++++++ .../vision/image/ImageCapture.java | 12 +++++++ .../vision/image/StaticImageCapture.java | 32 +++++++++++++++++++ .../vision/pipeline/CVPipeline.java | 12 +++---- .../vision/pipeline/CVPipeline2d.java | 14 ++++---- .../vision/pipeline/DriverVisionPipeline.java | 2 +- .../vision/pipeline/pipes/BlurPipe.java | 17 +++++----- .../pipeline/pipes/Collect2dTargetsPipe.java | 8 ++--- .../pipeline/pipes/Draw2dContoursPipe.java | 6 ++-- .../pipeline/pipes/FilterContoursPipe.java | 8 ++--- .../pipeline/pipes/SortContoursPipe.java | 8 ++--- .../chameleonvision/web/ServerHandler.java | 23 ++++++------- 23 files changed, 165 insertions(+), 130 deletions(-) rename Main/src/main/java/com/chameleonvision/vision/camera/{CameraProcess.java => CameraCapture.java} (67%) rename Main/src/main/java/com/chameleonvision/vision/camera/{CameraStaticProperties.java => CaptureStaticProperties.java} (92%) rename Main/src/main/java/com/chameleonvision/vision/camera/{USBCameraProcess.java => USBCameraCapture.java} (94%) create mode 100644 Main/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java create mode 100644 Main/src/main/java/com/chameleonvision/vision/image/ImageCapture.java create mode 100644 Main/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 7ef49e681..5cbae1d91 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -12,24 +12,13 @@ - - - - - - - - - - - - - - - - - - + + + + + + + diff --git a/Main/src/main/java/com/chameleonvision/Main.java b/Main/src/main/java/com/chameleonvision/Main.java index ff32ba8d0..9bc4d4da7 100644 --- a/Main/src/main/java/com/chameleonvision/Main.java +++ b/Main/src/main/java/com/chameleonvision/Main.java @@ -24,7 +24,7 @@ public class Main { private static final String IGNORE_ROOT_KEY = "--ignore-root"; // no args for this setting private static final String TEST_MODE_KEY = "--cv-development"; - private static final int DEFAULT_PORT = 5800; + private static final int DEFAULT_PORT = 8888; private static boolean ntServerMode = false; private static boolean manageNetwork = true; diff --git a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java index e4162bec3..b416eb65d 100644 --- a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java +++ b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java @@ -1,6 +1,6 @@ package com.chameleonvision.config; -import com.chameleonvision.vision.camera.USBCameraProcess; +import com.chameleonvision.vision.camera.USBCameraCapture; import com.chameleonvision.vision.camera.USBCameraProperties; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -30,7 +30,7 @@ public class CameraConfig { this.nickname = name; } - public static CameraConfig fromUSBCameraProcess(USBCameraProcess process) { + public static CameraConfig fromUSBCameraProcess(USBCameraCapture process) { USBCameraProperties camProps = process.getProperties(); return new CameraConfig(camProps.FOV, camProps.name, camProps.path, camProps.getNickname()); } diff --git a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java index dc5b30660..68212fc78 100644 --- a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java +++ b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java @@ -4,6 +4,7 @@ import com.chameleonvision.util.ProgramDirectoryUtilities; import com.chameleonvision.util.FileHelper; import com.chameleonvision.vision.pipeline.CVPipelineSettings; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -34,7 +35,8 @@ public class ConfigManager { } private static void checkSettingsFile() { - if (!settingsFileExists()) { + boolean settingsFileEmpty = settingsFileExists() && new File(settingsFilePath.toString()).length() == 0; + if (settingsFileEmpty || !settingsFileExists()) { try { FileHelper.Serializer(settingsFilePath, settings); } catch (IOException e) { diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java index b74708265..b0b148796 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java @@ -4,8 +4,8 @@ import com.chameleonvision.config.CameraConfig; import com.chameleonvision.config.ConfigManager; import com.chameleonvision.util.Helpers; import com.chameleonvision.util.Platform; -import com.chameleonvision.vision.camera.CameraProcess; -import com.chameleonvision.vision.camera.USBCameraProcess; +import com.chameleonvision.vision.camera.CameraCapture; +import com.chameleonvision.vision.camera.USBCameraCapture; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.UsbCameraInfo; @@ -86,7 +86,7 @@ public class VisionManager { public static boolean initializeProcesses() { for (int i = 0; i < loadedCameraConfigs.size(); i++) { CameraConfig config = loadedCameraConfigs.get(i); - CameraProcess camera = new USBCameraProcess(config); + CameraCapture camera = new USBCameraCapture(config); VisionProcess process = new VisionProcess(camera, config.name); visionProcesses.add(new VisionProcessManageable(i, config.name, process)); } @@ -136,7 +136,7 @@ public class VisionManager { String cameraName = process.getCamera().getProperties().name; List pipelines = process.getPipelines().stream().map(cvPipeline -> cvPipeline.settings).collect(Collectors.toList()); CVPipelineSettings driverMode = process.getDriverModeSettings(); - CameraConfig config = CameraConfig.fromUSBCameraProcess((USBCameraProcess) process.getCamera()); + CameraConfig config = CameraConfig.fromUSBCameraProcess((USBCameraCapture) process.getCamera()); try { ConfigManager.saveCameraPipelines(cameraName, pipelines); ConfigManager.saveCameraDriverMode(cameraName, driverMode); diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index 3cb5c8ad1..30589dc71 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -2,7 +2,7 @@ package com.chameleonvision.vision; import com.chameleonvision.config.ConfigManager; import com.chameleonvision.util.LoopingRunnable; -import com.chameleonvision.vision.camera.CameraProcess; +import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.vision.camera.CameraStreamer; import com.chameleonvision.vision.pipeline.*; import com.chameleonvision.web.ServerHandler; @@ -19,7 +19,7 @@ import java.util.List; public class VisionProcess { - private final CameraProcess cameraProcess; + private final CameraCapture cameraCapture; private final List pipelines = new ArrayList<>(); private final CameraFrameRunnable cameraRunnable; private final CameraStreamerRunnable streamRunnable; @@ -51,29 +51,29 @@ public class VisionProcess { private NetworkTableEntry ntValidEntry; private Gson gson = new GsonBuilder().setPrettyPrinting().create(); - VisionProcess(CameraProcess cameraProcess, String name) { - this.cameraProcess = cameraProcess; + VisionProcess(CameraCapture cameraCapture, String name) { + this.cameraCapture = cameraCapture; pipelines.add(new CVPipeline2d("New Pipeline")); setPipeline(0, false); // Thread to grab frames from the camera // TODO: (HIGH) fix video modes!!! - this.cameraRunnable = new CameraFrameRunnable(cameraProcess.getProperties().videoModes.get(0).fps); + this.cameraRunnable = new CameraFrameRunnable(cameraCapture.getProperties().videoModes.get(0).fps); lastPipelineResult = new DriverVisionPipeline.DriverPipelineResult( null, cameraRunnable.getFrame(new Mat()), 0 ); // Thread to put frames on the dashboard - this.cameraStreamer = new CameraStreamer(cameraProcess, name); + this.cameraStreamer = new CameraStreamer(cameraCapture, name); this.streamRunnable = new CameraStreamerRunnable(30, cameraStreamer); // Thread to process vision data this.visionRunnable = new VisionProcessRunnable(); // network table - defaultTable = NetworkTableInstance.getDefault().getTable("/chameleon-vision/" + cameraProcess.getProperties().name); + defaultTable = NetworkTableInstance.getDefault().getTable("/chameleon-vision/" + cameraCapture.getProperties().name); } public void start() { @@ -156,7 +156,7 @@ public class VisionProcess { currentPipelineIndex = pipelineIndex; // update the configManager - if(ConfigManager.settings.currentCamera.equals(cameraProcess.getProperties().name)) { + if(ConfigManager.settings.currentCamera.equals(cameraCapture.getProperties().name)) { ConfigManager.settings.currentPipeline = pipelineIndex; if (updateUI) { @@ -171,11 +171,11 @@ public class VisionProcess { private void setPipelineInternal(CVPipeline pipeline) { currentPipeline = pipeline; - currentPipeline.initPipeline(cameraProcess); + currentPipeline.initPipeline(cameraCapture); } private void updateUI(CVPipelineResult data) { - if(cameraProcess.getProperties().name.equals(ConfigManager.settings.currentCamera)) { + if(cameraCapture.getProperties().name.equals(ConfigManager.settings.currentCamera)) { HashMap WebSend = new HashMap<>(); HashMap point = new HashMap<>(); HashMap calculated = new HashMap<>(); @@ -235,7 +235,7 @@ public class VisionProcess { } public void setVideoMode(VideoMode newMode) { - cameraProcess.setVideoMode(newMode); + cameraCapture.setVideoMode(newMode); cameraRunnable.updateCameraFPS(newMode.fps); cameraStreamer.setNewVideoMode(newMode); } @@ -261,8 +261,8 @@ public class VisionProcess { pipelines.add(pipeline); } - public CameraProcess getCamera() { - return cameraProcess; + public CameraCapture getCamera() { + return cameraCapture; } public boolean getDriverMode() { @@ -304,10 +304,9 @@ public class VisionProcess { @Override public void process() { -// System.out.println("running camera grabber process"); // Grab camera frames - var camData = cameraProcess.getFrame(); + var camData = cameraCapture.getFrame(); if (camData.getLeft().cols() > 0) { // System.out.println("grabbing frame"); // synchronized (frameLock) { diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraProcess.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java similarity index 67% rename from Main/src/main/java/com/chameleonvision/vision/camera/CameraProcess.java rename to Main/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java index ef9d4a7bb..2233eb2cc 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java @@ -1,18 +1,11 @@ package com.chameleonvision.vision.camera; +import com.chameleonvision.vision.image.ImageCapture; import edu.wpi.cscore.VideoMode; -import org.apache.commons.lang3.tuple.Pair; -import org.opencv.core.Mat; -public interface CameraProcess { +public interface CameraCapture extends ImageCapture { USBCameraProperties getProperties(); - /** - * Get the next camera frame - * @return a Pair of the captured image and how long it took to grab the frame (in uS) - */ - Pair getFrame(); - /** * Set the exposure of the camera * @param exposure the new exposure to set the camera to diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java index 249e0f425..1a2e68aa6 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java @@ -10,32 +10,32 @@ import org.opencv.core.CvType; import org.opencv.core.Mat; public class CameraStreamer { - private final CameraProcess cameraProcess; + private final CameraCapture cameraCapture; private final String name; private StreamDivisor divisor = StreamDivisor.NONE; private CvSource cvSource; private final Object streamBufferLock = new Object(); private Mat streamBuffer = new Mat(); - public CameraStreamer(CameraProcess cameraProcess, String name) { - this.cameraProcess = cameraProcess; + public CameraStreamer(CameraCapture cameraCapture, String name) { + this.cameraCapture = cameraCapture; this.name = name; this.cvSource = CameraServer.getInstance().putVideo(name, - cameraProcess.getProperties().staticProperties.imageWidth / divisor.value, - cameraProcess.getProperties().staticProperties.imageHeight / divisor.value); + cameraCapture.getProperties().getStaticProperties().imageWidth / divisor.value, + cameraCapture.getProperties().getStaticProperties().imageHeight / divisor.value); setDivisor(divisor, false); } public void setDivisor(StreamDivisor newDivisor, boolean updateUI) { this.divisor = newDivisor; - var camValues = cameraProcess.getProperties(); - var newWidth = camValues.staticProperties.imageWidth / newDivisor.value; - var newHeight = camValues.staticProperties.imageHeight / newDivisor.value; + var camValues = cameraCapture.getProperties(); + var newWidth = camValues.getStaticProperties().imageWidth / newDivisor.value; + var newHeight = camValues.getStaticProperties().imageHeight / newDivisor.value; synchronized (streamBufferLock) { this.streamBuffer = new Mat(newWidth, newHeight, CvType.CV_8UC3); this.cvSource = CameraServer.getInstance().putVideo(this.name, - cameraProcess.getProperties().staticProperties.imageWidth / divisor.value, - cameraProcess.getProperties().staticProperties.imageHeight / divisor.value); + cameraCapture.getProperties().getStaticProperties().imageWidth / divisor.value, + cameraCapture.getProperties().getStaticProperties().imageHeight / divisor.value); } if (updateUI) { ServerHandler.sendFullSettings(); diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraStaticProperties.java b/Main/src/main/java/com/chameleonvision/vision/camera/CaptureStaticProperties.java similarity index 92% rename from Main/src/main/java/com/chameleonvision/vision/camera/CameraStaticProperties.java rename to Main/src/main/java/com/chameleonvision/vision/camera/CaptureStaticProperties.java index 5ab0e17d8..4c991f5d6 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraStaticProperties.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CaptureStaticProperties.java @@ -3,7 +3,7 @@ package com.chameleonvision.vision.camera; import org.apache.commons.math3.fraction.Fraction; import org.apache.commons.math3.util.FastMath; -public class CameraStaticProperties { +public class CaptureStaticProperties { public final int imageWidth; public final int imageHeight; @@ -14,7 +14,7 @@ public class CameraStaticProperties { public final double horizontalFocalLength; public final double verticalFocalLength; - public CameraStaticProperties(int imageWidth, int imageHeight, double fov) { + public CaptureStaticProperties(int imageWidth, int imageHeight, double fov) { this.imageWidth = imageWidth; this.imageHeight = imageHeight; this.fov = fov; diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProcess.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java similarity index 94% rename from Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProcess.java rename to Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java index 4db1bcab7..b3391bee8 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java @@ -9,13 +9,13 @@ import edu.wpi.first.cameraserver.CameraServer; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; -public class USBCameraProcess implements CameraProcess { +public class USBCameraCapture implements CameraCapture { private final UsbCamera baseCamera; private final CvSink cvSink; private Mat imageBuffer = new Mat(); private USBCameraProperties properties; - public USBCameraProcess(CameraConfig config) { + public USBCameraCapture(CameraConfig config) { baseCamera = new UsbCamera(config.name, config.path); cvSink = CameraServer.getInstance().getVideo(baseCamera); properties = new USBCameraProperties(baseCamera, config); diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java index 28e27b785..e91fe825b 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java @@ -2,9 +2,9 @@ package com.chameleonvision.vision.camera; import com.chameleonvision.config.CameraConfig; import com.chameleonvision.util.Platform; +import com.chameleonvision.vision.image.CaptureProperties; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.VideoMode; -import org.apache.commons.math3.util.FastMath; import java.util.Arrays; import java.util.List; @@ -12,7 +12,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -public class USBCameraProperties { +public class USBCameraProperties extends CaptureProperties { public static final double DEFAULT_FOV = 70; private static final int DEFAULT_EXPOSURE = 50; private static final int DEFAULT_BRIGHTNESS = 50; @@ -26,19 +26,18 @@ public class USBCameraProperties { private static final Predicate kMinSizePredicate = (videoMode -> videoMode.width >= MINIMUM_WIDTH && videoMode.height >= MINIMUM_HEIGHT); private static final Predicate kPixelFormatPredicate = (videoMode -> ALLOWED_PIXEL_FORMATS.contains(videoMode.pixelFormat)); - public CameraStaticProperties staticProperties; public final String name; public final String path; public final List videoModes; private final UsbCamera baseCamera; + private final boolean hasGain; private String nickname; public double FOV; - public final boolean hasGain; - public USBCameraProperties(UsbCamera baseCamera, CameraConfig config) { + USBCameraProperties(UsbCamera baseCamera, CameraConfig config) { FOV = config.fov; name = config.name; path = config.path; @@ -86,17 +85,8 @@ public class USBCameraProperties { return validModes.collect(Collectors.toList()); } - public void updateVideoMode(VideoMode videoMode) { - staticProperties = new CameraStaticProperties(videoMode.width, videoMode.height, FOV); - } - - public double calculatePitch(double PixelY, double centerY) { - double pitch = FastMath.toDegrees(FastMath.atan((PixelY - centerY) / staticProperties.verticalFocalLength)); - return (pitch * -1); - } - - public double calculateYaw(double PixelX, double centerX) { - return FastMath.toDegrees(FastMath.atan((PixelX - centerX) / staticProperties.horizontalFocalLength)); + void updateVideoMode(VideoMode videoMode) { + staticProperties = new CaptureStaticProperties(videoMode.width, videoMode.height, FOV); } public List getVideoModes() { diff --git a/Main/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java b/Main/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java new file mode 100644 index 000000000..139023fcd --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java @@ -0,0 +1,20 @@ +package com.chameleonvision.vision.image; + +import com.chameleonvision.vision.camera.CaptureStaticProperties; +import org.opencv.core.Mat; + +public class CaptureProperties { + + protected CaptureStaticProperties staticProperties; + + protected CaptureProperties() { + } + + public CaptureProperties(Mat staticImage, double fov) { + staticProperties = new CaptureStaticProperties(staticImage.cols(), staticImage.rows(), fov); + } + + public CaptureStaticProperties getStaticProperties() { + return staticProperties; + } +} diff --git a/Main/src/main/java/com/chameleonvision/vision/image/ImageCapture.java b/Main/src/main/java/com/chameleonvision/vision/image/ImageCapture.java new file mode 100644 index 000000000..1d7b3820e --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/image/ImageCapture.java @@ -0,0 +1,12 @@ +package com.chameleonvision.vision.image; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; + +public interface ImageCapture { + /** + * Get the next camera frame + * @return a Pair of the captured image and the Linux epoch of when the frame was grabbed (in uS) + */ + Pair getFrame(); +} diff --git a/Main/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java b/Main/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java new file mode 100644 index 000000000..3f738f118 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java @@ -0,0 +1,32 @@ +package com.chameleonvision.vision.image; + +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; +import org.opencv.imgcodecs.Imgcodecs; + +import java.nio.file.Files; +import java.nio.file.Path; + +public class StaticImageCapture implements ImageCapture { + + private final Mat image = new Mat(); + + public StaticImageCapture(Path imagePath) { + if (!Files.exists(imagePath)) throw new RuntimeException("Invalid path for image!"); + + Mat tempMat = new Mat(); + + try { + tempMat = Imgcodecs.imread(imagePath.toString()); + } catch (Exception e) { + System.err.println("Failed to read image!"); + } finally { + tempMat.copyTo(image); + } + } + + @Override + public Pair getFrame() { + return Pair.of(image, System.nanoTime()); + } +} diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java index a56c03e47..9b27f65d2 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java @@ -1,6 +1,6 @@ package com.chameleonvision.vision.pipeline; -import com.chameleonvision.vision.camera.CameraProcess; +import com.chameleonvision.vision.camera.CameraCapture; import org.opencv.core.Mat; /** @@ -9,7 +9,7 @@ import org.opencv.core.Mat; */ public abstract class CVPipeline { protected Mat outputMat = new Mat(); - CameraProcess cameraProcess; + CameraCapture cameraCapture; public final S settings; protected CVPipeline(S settings) { @@ -21,10 +21,10 @@ public abstract class CVPipeline { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - input.copyTo(processBuffer); - - try { - if (blurSize > 0) { + if (blurSize > 0) { + input.copyTo(processBuffer); + try { Imgproc.blur(processBuffer, processBuffer, new Size(blurSize, blurSize)); + processBuffer.copyTo(outputMat); + } catch (CvException e) { + System.err.println("(BlurPipe) Exception thrown by OpenCV: \n" + e.getMessage()); } - } catch (CvException e) { - System.err.println("(BlurPipe) Exception thrown by OpenCV: \n" + e.getMessage()); + } else { + input.copyTo(outputMat); } + long processTime = System.nanoTime() - processStartNanos; - processBuffer.copyTo(outputMat); Pair output = Pair.of(outputMat, processTime); processBuffer.release(); - return output; } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java index 51747cfdd..87a627fc2 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java @@ -1,6 +1,6 @@ package com.chameleonvision.vision.pipeline.pipes; -import com.chameleonvision.vision.camera.CameraStaticProperties; +import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.vision.pipeline.CVPipeline2d; import com.chameleonvision.vision.enums.CalibrationMode; import org.apache.commons.lang3.tuple.Pair; @@ -13,14 +13,14 @@ import java.util.List; public class Collect2dTargetsPipe implements Pipe, List> { private CalibrationMode calibrationMode; - private CameraStaticProperties camProps; + private CaptureStaticProperties camProps; private List calibrationPoint; private double calibrationM, calibrationB; private List targets = new ArrayList<>(); public Collect2dTargetsPipe(CalibrationMode calibrationMode, List calibrationPoint, - double calibrationM, double calibrationB, CameraStaticProperties camProps) { + double calibrationM, double calibrationB, CaptureStaticProperties camProps) { this.calibrationMode = calibrationMode; this.camProps = camProps; this.calibrationPoint = calibrationPoint; @@ -29,7 +29,7 @@ public class Collect2dTargetsPipe implements Pipe, List calibrationPoint, - double calibrationM, double calibrationB, CameraStaticProperties camProps) { + double calibrationM, double calibrationB, CaptureStaticProperties camProps) { this.calibrationMode = calibrationMode; this.camProps = camProps; this.calibrationPoint = calibrationPoint; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java index ebbab7d1b..6b95ca930 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java @@ -1,6 +1,6 @@ package com.chameleonvision.vision.pipeline.pipes; -import com.chameleonvision.vision.camera.CameraStaticProperties; +import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.util.Helpers; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Point; @@ -14,11 +14,11 @@ import java.util.List; public class Draw2dContoursPipe implements Pipe>, Mat> { private final Draw2dContoursSettings settings; - private final CameraStaticProperties camProps; + private final CaptureStaticProperties camProps; private Mat outputMat = new Mat(); - public Draw2dContoursPipe(Draw2dContoursSettings settings, CameraStaticProperties camProps) { + public Draw2dContoursPipe(Draw2dContoursSettings settings, CaptureStaticProperties camProps) { this.settings = settings; this.camProps = camProps; } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java index 97acea6aa..9870e43a5 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java @@ -1,6 +1,6 @@ package com.chameleonvision.vision.pipeline.pipes; -import com.chameleonvision.vision.camera.CameraStaticProperties; +import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.util.MathHandler; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.MatOfPoint; @@ -16,18 +16,18 @@ public class FilterContoursPipe implements Pipe, List area; private List ratio; private List extent; - private CameraStaticProperties camProps; + private CaptureStaticProperties camProps; private List filteredContours = new ArrayList<>(); - public FilterContoursPipe(List area, List ratio, List extent, CameraStaticProperties camProps) { + public FilterContoursPipe(List area, List ratio, List extent, CaptureStaticProperties camProps) { this.area = area; this.ratio = ratio; this.extent = extent; this.camProps = camProps; } - public void setConfig(List area, List ratio, List extent, CameraStaticProperties camProps) { + public void setConfig(List area, List ratio, List extent, CaptureStaticProperties camProps) { this.area = area; this.ratio = ratio; this.extent = extent; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java index a6e28f1c7..329b848a2 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java @@ -1,6 +1,6 @@ package com.chameleonvision.vision.pipeline.pipes; -import com.chameleonvision.vision.camera.CameraStaticProperties; +import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.vision.enums.SortMode; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.math3.util.FastMath; @@ -25,16 +25,16 @@ public class SortContoursPipe implements Pipe, List sortedContours = new ArrayList<>(); - public SortContoursPipe(SortMode sort, CameraStaticProperties camProps) { + public SortContoursPipe(SortMode sort, CaptureStaticProperties camProps) { this.sort = sort; this.camProps = camProps; } - public void setConfig(SortMode sort, CameraStaticProperties camProps) { + public void setConfig(SortMode sort, CaptureStaticProperties camProps) { this.sort = sort; this.camProps = camProps; } diff --git a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java index 35006e66e..b3608b02d 100644 --- a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java +++ b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java @@ -3,7 +3,7 @@ package com.chameleonvision.web; import com.chameleonvision.config.GeneralSettings; import com.chameleonvision.vision.VisionManager; import com.chameleonvision.vision.VisionProcess; -import com.chameleonvision.vision.camera.CameraProcess; +import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.config.ConfigManager; import com.chameleonvision.vision.enums.CalibrationMode; import com.chameleonvision.vision.pipeline.CVPipeline; @@ -38,22 +38,23 @@ public class ServerHandler { sendFullSettings(); } - public void onClose(WsCloseContext context) { + void onClose(WsCloseContext context) { users.remove(context); } + @SuppressWarnings("unchecked") void onBinaryMessage(WsBinaryMessageContext context) throws Exception { - Map deserialized = objectMapper.readValue(ArrayUtils.toPrimitive(context.data()), new TypeReference>() { + Map deserialized = objectMapper.readValue(ArrayUtils.toPrimitive(context.data()), new TypeReference<>() { }); for (Map.Entry entry : deserialized.entrySet()) { try { - var data = (HashMap) entry.getValue(); VisionProcess currentProcess = VisionManager.getCurrentUIVisionProcess(); - CameraProcess currentCamera = currentProcess.getCamera(); + CameraCapture currentCamera = currentProcess.getCamera(); CVPipeline currentPipeline = currentProcess.getCurrentPipeline(); switch (entry.getKey()) { case "generalSettings": { + HashMap data = (HashMap) entry.getValue(); for (HashMap.Entry e : data.entrySet()) { setField(ConfigManager.settings, e.getKey(), e.getValue()); } @@ -62,6 +63,7 @@ public class ServerHandler { break; } case "driverMode": { + HashMap data = (HashMap) entry.getValue(); currentProcess.getDriverModeSettings().exposure = (Integer) data.get("exposure"); currentProcess.getDriverModeSettings().brightness = (Integer) data.get("brightness"); currentProcess.setDriverMode((Boolean) data.get("isDriver")); @@ -74,7 +76,7 @@ public class ServerHandler { Number newFOV = (Number) camSettings.get("fov"); StreamDivisor newStreamDivisor = StreamDivisor.values()[(Integer) camSettings.get("streamDivisor")]; - Integer newResolution = (Integer) camSettings.get("resolution"); +// Integer newResolution = (Integer) camSettings.get("resolution"); currentCamera.getProperties().FOV = (double) newFOV; @@ -119,7 +121,6 @@ public class ServerHandler { } else { currentProcess.addPipeline(origPipeline); } - // TODO: (HIGH) switch to ConfigManager ConfigManager.saveSettings(); break; } @@ -128,7 +129,6 @@ public class ServerHandler { case "addNewPipeline": currentProcess.addPipeline(); sendFullSettings(); - // TODO: (HIGH) switch to ConfigManager ConfigManager.saveSettings(); break; // TODO: (HIGH) this never worked before, re-visit now that VisionProcess is written sanely @@ -146,7 +146,6 @@ public class ServerHandler { // ConfigManager.saveSettings(); break; case "save": - // TODO: (HIGH) switch to ConfigManager ConfigManager.saveSettings(); System.out.println("saved Settings"); break; @@ -155,7 +154,6 @@ public class ServerHandler { break; } case "currentCamera": { - // TODO: (HIGH) find way to map cameras to indexes VisionManager.setCurrentProcessByIndex((Integer) entry.getValue()); sendFullSettings(); break; @@ -254,7 +252,7 @@ public class ServerHandler { private static HashMap getOrdinalCameraSettings() { HashMap tmp = new HashMap<>(); VisionProcess currentVisionProcess = VisionManager.getCurrentUIVisionProcess(); - CameraProcess currentCamera = VisionManager.getCurrentUIVisionProcess().getCamera(); + CameraCapture currentCamera = VisionManager.getCurrentUIVisionProcess().getCamera(); tmp.put("fov", currentCamera.getProperties().FOV); tmp.put("streamDivisor", currentVisionProcess.cameraStreamer.getDivisor().ordinal()); // TODO: (HIGH) get videomode index! @@ -316,7 +314,7 @@ public class ServerHandler { Map fullSettings = new HashMap<>(); VisionProcess currentProcess = VisionManager.getCurrentUIVisionProcess(); - CameraProcess currentCamera = currentProcess.getCamera(); + CameraCapture currentCamera = currentProcess.getCamera(); CVPipeline currentPipeline = currentProcess.getCurrentPipeline(); try { @@ -327,7 +325,6 @@ public class ServerHandler { fullSettings.put("cameraList", VisionManager.getAllCameraNicknames()); fullSettings.put("pipeline", getOrdinalPipeline()); fullSettings.put("driverMode", getOrdinalDriver()); - // TODO (HIGH) all of these settings! fullSettings.put("pipelineList", VisionManager.getCurrentCameraPipelineNicknames()); fullSettings.put("resolutionList", VisionManager.getCurrentCameraResolutionList()); fullSettings.put("FOV", currentCamera.getProperties().FOV); From 55cc337198b7a2edd88d710fb80e3dc4b5ee18e5 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Mon, 25 Nov 2019 16:06:21 -0500 Subject: [PATCH 69/84] tweaked pipeline FPS prints --- .../vision/pipeline/CVPipeline2d.java | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index fea3379c8..b895cb399 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -63,8 +63,8 @@ public class CVPipeline2d extends CVPipeline rotateFlipResult = rotateFlipPipe.run(inputMat); - totalProcessTimeNanos += rotateFlipResult.getRight(); - procTimeStringBuilder.append(String.format("RotateFlip: %.2fms, ", rotateFlipResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += rotateFlipResult.getRight(); + pipelineTimeStringBuilder.append(String.format("RotateFlip: %.2fms, ", rotateFlipResult.getRight() / 1000000.0)); Pair blurResult = blurPipe.run(rotateFlipResult.getLeft()); - totalProcessTimeNanos += blurResult.getRight(); - procTimeStringBuilder.append(String.format("Blur: %.2fms, ", blurResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += blurResult.getRight(); + pipelineTimeStringBuilder.append(String.format("Blur: %.2fms, ", blurResult.getRight() / 1000000.0)); Pair erodeDilateResult = erodeDilatePipe.run(blurResult.getLeft()); - totalProcessTimeNanos += erodeDilateResult.getRight(); - procTimeStringBuilder.append(String.format("ErodeDilate: %.2fms, ", erodeDilateResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += erodeDilateResult.getRight(); + pipelineTimeStringBuilder.append(String.format("ErodeDilate: %.2fms, ", erodeDilateResult.getRight() / 1000000.0)); Pair hsvResult = hsvPipe.run(erodeDilateResult.getLeft()); - totalProcessTimeNanos += hsvResult.getRight(); + totalPipelineTimeNanos += hsvResult.getRight(); try { Imgproc.cvtColor(hsvResult.getLeft(), hsvOutputMat, Imgproc.COLOR_GRAY2BGR, 3); @@ -131,46 +131,51 @@ public class CVPipeline2d extends CVPipeline, Long> findContoursResult = findContoursPipe.run(hsvResult.getLeft()); - totalProcessTimeNanos += findContoursResult.getRight(); - procTimeStringBuilder.append(String.format("FindContours: %.2fms, ", findContoursResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += findContoursResult.getRight(); + pipelineTimeStringBuilder.append(String.format("FindContours: %.2fms, ", findContoursResult.getRight() / 1000000.0)); Pair, Long> filterContoursResult = filterContoursPipe.run(findContoursResult.getLeft()); - totalProcessTimeNanos += filterContoursResult.getRight(); - procTimeStringBuilder.append(String.format("FilterContours: %.2fms, ", filterContoursResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += filterContoursResult.getRight(); + pipelineTimeStringBuilder.append(String.format("FilterContours: %.2fms, ", filterContoursResult.getRight() / 1000000.0)); Pair, Long> speckleRejectResult = speckleRejectPipe.run(filterContoursResult.getLeft()); - totalProcessTimeNanos += speckleRejectResult.getRight(); - procTimeStringBuilder.append(String.format("SpeckleReject: %.2fms, ", speckleRejectResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += speckleRejectResult.getRight(); + pipelineTimeStringBuilder.append(String.format("SpeckleReject: %.2fms, ", speckleRejectResult.getRight() / 1000000.0)); Pair, Long> groupContoursResult = groupContoursPipe.run(speckleRejectResult.getLeft()); - totalProcessTimeNanos += groupContoursResult.getRight(); - procTimeStringBuilder.append(String.format("GroupContours: %.2fms, ", groupContoursResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += groupContoursResult.getRight(); + pipelineTimeStringBuilder.append(String.format("GroupContours: %.2fms, ", groupContoursResult.getRight() / 1000000.0)); Pair, Long> sortContoursResult = sortContoursPipe.run(groupContoursResult.getLeft()); - totalProcessTimeNanos += sortContoursResult.getRight(); - procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += sortContoursResult.getRight(); + pipelineTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); Pair, Long> collect2dTargetsResult = collect2dTargetsPipe.run(sortContoursResult.getLeft()); - totalProcessTimeNanos += collect2dTargetsResult.getRight(); - procTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += collect2dTargetsResult.getRight(); + pipelineTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); // takes pair of (Mat of original camera image, Mat of HSV thresholded image) Pair outputMatResult = outputMatPipe.run(Pair.of(rawCameraMat, hsvOutputMat)); - totalProcessTimeNanos += outputMatResult.getRight(); - procTimeStringBuilder.append(String.format("OutputMat: %.2fms, ", outputMatResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += outputMatResult.getRight(); + pipelineTimeStringBuilder.append(String.format("OutputMat: %.2fms, ", outputMatResult.getRight() / 1000000.0)); // takes pair of (Mat to draw on, List of sorted contours) Pair draw2dContoursResult = draw2dContoursPipe.run(Pair.of(outputMatResult.getLeft(), sortContoursResult.getLeft())); - totalProcessTimeNanos += draw2dContoursResult.getRight(); - procTimeStringBuilder.append(String.format("Draw2dContours: %.2fms, ", draw2dContoursResult.getRight() / 1000000.0)); + totalPipelineTimeNanos += draw2dContoursResult.getRight(); + pipelineTimeStringBuilder.append(String.format("Draw2dContours: %.2fms, ", draw2dContoursResult.getRight() / 1000000.0)); - System.out.println(procTimeStringBuilder.toString()); - System.out.printf("Pipeline ran in %.3fms\n", totalProcessTimeNanos / 1000000.0); + System.out.println(pipelineTimeStringBuilder.toString()); + double totalPipelineTimeMillis = totalPipelineTimeNanos / 1000000.0; + double totalPipelineTimeFPS = 1.0 / (totalPipelineTimeMillis / 1000.0); + double truePipelineTimeMillis = (System.nanoTime() - pipelineStartTimeNanos) / 1000000.0; + double truePipelineFPS = 1.0 / (truePipelineTimeMillis / 1000.0); + System.out.printf("Pipeline processed in %.3fms (%.2fFPS), ", totalPipelineTimeMillis, totalPipelineTimeFPS); + System.out.printf("full pipeline run time was %.3fms (%.2fFPS)\n", truePipelineTimeMillis, truePipelineFPS); - return new CVPipeline2dResult(collect2dTargetsResult.getLeft(), draw2dContoursResult.getLeft(), totalProcessTimeNanos); + return new CVPipeline2dResult(collect2dTargetsResult.getLeft(), draw2dContoursResult.getLeft(), totalPipelineTimeNanos); } public static class CVPipeline2dResult extends CVPipelineResult { From 79fd79be415548f80c0e439e96eca3cc8165fbe7 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Mon, 25 Nov 2019 16:40:23 -0500 Subject: [PATCH 70/84] Memory cleanups --- .../vision/pipeline/CVPipeline2d.java | 75 +++++++++++-------- .../pipeline/pipes/Draw2dContoursPipe.java | 7 +- .../vision/pipeline/pipes/OutputMatPipe.java | 4 + 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index b895cb399..8bb2f3f51 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -17,6 +17,7 @@ public class CVPipeline2d extends CVPipeline rotateFlipResult = rotateFlipPipe.run(inputMat); totalPipelineTimeNanos += rotateFlipResult.getRight(); - pipelineTimeStringBuilder.append(String.format("RotateFlip: %.2fms, ", rotateFlipResult.getRight() / 1000000.0)); Pair blurResult = blurPipe.run(rotateFlipResult.getLeft()); totalPipelineTimeNanos += blurResult.getRight(); - pipelineTimeStringBuilder.append(String.format("Blur: %.2fms, ", blurResult.getRight() / 1000000.0)); Pair erodeDilateResult = erodeDilatePipe.run(blurResult.getLeft()); totalPipelineTimeNanos += erodeDilateResult.getRight(); - pipelineTimeStringBuilder.append(String.format("ErodeDilate: %.2fms, ", erodeDilateResult.getRight() / 1000000.0)); Pair hsvResult = hsvPipe.run(erodeDilateResult.getLeft()); totalPipelineTimeNanos += hsvResult.getRight(); + // Todo: move to a pipe try { Imgproc.cvtColor(hsvResult.getLeft(), hsvOutputMat, Imgproc.COLOR_GRAY2BGR, 3); } catch (CvException e) { System.err.println("(CVPipeline2d) Exception thrown by OpenCV: \n" + e.getMessage()); } - pipelineTimeStringBuilder.append(String.format("HSV: %.2fms, ", hsvResult.getRight() / 1000000.0)); - Pair, Long> findContoursResult = findContoursPipe.run(hsvResult.getLeft()); totalPipelineTimeNanos += findContoursResult.getRight(); - pipelineTimeStringBuilder.append(String.format("FindContours: %.2fms, ", findContoursResult.getRight() / 1000000.0)); Pair, Long> filterContoursResult = filterContoursPipe.run(findContoursResult.getLeft()); totalPipelineTimeNanos += filterContoursResult.getRight(); - pipelineTimeStringBuilder.append(String.format("FilterContours: %.2fms, ", filterContoursResult.getRight() / 1000000.0)); Pair, Long> speckleRejectResult = speckleRejectPipe.run(filterContoursResult.getLeft()); totalPipelineTimeNanos += speckleRejectResult.getRight(); - pipelineTimeStringBuilder.append(String.format("SpeckleReject: %.2fms, ", speckleRejectResult.getRight() / 1000000.0)); Pair, Long> groupContoursResult = groupContoursPipe.run(speckleRejectResult.getLeft()); totalPipelineTimeNanos += groupContoursResult.getRight(); - pipelineTimeStringBuilder.append(String.format("GroupContours: %.2fms, ", groupContoursResult.getRight() / 1000000.0)); Pair, Long> sortContoursResult = sortContoursPipe.run(groupContoursResult.getLeft()); totalPipelineTimeNanos += sortContoursResult.getRight(); - pipelineTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); Pair, Long> collect2dTargetsResult = collect2dTargetsPipe.run(sortContoursResult.getLeft()); totalPipelineTimeNanos += collect2dTargetsResult.getRight(); - pipelineTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); // takes pair of (Mat of original camera image, Mat of HSV thresholded image) Pair outputMatResult = outputMatPipe.run(Pair.of(rawCameraMat, hsvOutputMat)); totalPipelineTimeNanos += outputMatResult.getRight(); - pipelineTimeStringBuilder.append(String.format("OutputMat: %.2fms, ", outputMatResult.getRight() / 1000000.0)); // takes pair of (Mat to draw on, List of sorted contours) Pair draw2dContoursResult = draw2dContoursPipe.run(Pair.of(outputMatResult.getLeft(), sortContoursResult.getLeft())); totalPipelineTimeNanos += draw2dContoursResult.getRight(); + + pipelineTimeStringBuilder.append(String.format("PipeInit: %.2fms, ", pipeInitTimeNanos / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("RotateFlip: %.2fms, ", rotateFlipResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("Blur: %.2fms, ", blurResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("ErodeDilate: %.2fms, ", erodeDilateResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("HSV: %.2fms, ", hsvResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("FindContours: %.2fms, ", findContoursResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("FilterContours: %.2fms, ", filterContoursResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("SpeckleReject: %.2fms, ", speckleRejectResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("GroupContours: %.2fms, ", groupContoursResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("SortContours: %.2fms, ", sortContoursResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("Collect2dTargets: %.2fms, ", collect2dTargetsResult.getRight() / 1000000.0)); + pipelineTimeStringBuilder.append(String.format("OutputMat: %.2fms, ", outputMatResult.getRight() / 1000000.0)); pipelineTimeStringBuilder.append(String.format("Draw2dContours: %.2fms, ", draw2dContoursResult.getRight() / 1000000.0)); System.out.println(pipelineTimeStringBuilder.toString()); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java index 6b95ca930..3fc361f47 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java @@ -2,6 +2,7 @@ package com.chameleonvision.vision.pipeline.pipes; import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.util.Helpers; +import com.chameleonvision.vision.image.CaptureProperties; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Point; import org.opencv.core.*; @@ -14,7 +15,7 @@ import java.util.List; public class Draw2dContoursPipe implements Pipe>, Mat> { private final Draw2dContoursSettings settings; - private final CaptureStaticProperties camProps; + private CaptureStaticProperties camProps; private Mat outputMat = new Mat(); @@ -23,6 +24,10 @@ public class Draw2dContoursPipe implements Pipe>, Ma this.camProps = camProps; } + public void setConfig(CaptureStaticProperties captureProps) { + camProps = captureProps; + } + @Override public Pair run(Pair> input) { long processStartNanos = System.nanoTime(); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java index 7593a2ca8..df7972108 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java @@ -13,6 +13,10 @@ public class OutputMatPipe implements Pipe, Mat> { this.showThresholded = showThresholded; } + public void setConfig(boolean showThresholded) { + this.showThresholded = showThresholded; + } + @Override public Pair run(Pair input) { long processStartNanos = System.nanoTime(); From 29b52d485b08a7fdb718bb257ecd9cf378b54547 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Tue, 26 Nov 2019 01:39:36 -0500 Subject: [PATCH 71/84] Add PS3Eye detection --- .../vision/camera/USBCameraProperties.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java index e91fe825b..56fd9521c 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java @@ -20,6 +20,10 @@ public class USBCameraProperties extends CaptureProperties { private static final int MINIMUM_WIDTH = 320; private static final int MINIMUM_HEIGHT = 200; private static final int MAX_INIT_MS = 1500; + + private static final int PS3EYE_VID = 1415; + private static final int PS3EYE_PID = 2000; + private static final List ALLOWED_PIXEL_FORMATS = Arrays.asList(VideoMode.PixelFormat.kYUYV, VideoMode.PixelFormat.kMJPEG); private static final Predicate kMinFPSPredicate = (videoMode -> videoMode.fps >= MINIMUM_FPS); @@ -31,12 +35,11 @@ public class USBCameraProperties extends CaptureProperties { public final List videoModes; private final UsbCamera baseCamera; - private final boolean hasGain; + public final boolean isPS3Eye; private String nickname; public double FOV; - USBCameraProperties(UsbCamera baseCamera, CameraConfig config) { FOV = config.fov; name = config.name; @@ -44,6 +47,9 @@ public class USBCameraProperties extends CaptureProperties { nickname = config.nickname; this.baseCamera = baseCamera; + int usbVID = baseCamera.getInfo().vendorId; + int usbPID = baseCamera.getInfo().productId; + // wait for camera USB init on Windows, Windows USB is slow... if (Platform.CurrentPlatform == Platform.WINDOWS_64 && !baseCamera.isConnected()) { System.out.print("Waiting on camera... "); @@ -57,17 +63,7 @@ public class USBCameraProperties extends CaptureProperties { System.out.printf("USBCameraProcess initialized in %.2fms\n", initTimeMs); } - // TODO: (low) find way to determine if camera is a PS3Eye - hasGain = false; -// var props = baseCamera.enumerateProperties(); -// for (var prop : props) { -// var name = prop.getName(); -// var min = prop.getMin(); -// var max = prop.getMax(); -// var _default = prop.getDefault(); -// var kind = prop.getKind(); -// } - + isPS3Eye = (usbVID == PS3EYE_VID && usbPID == PS3EYE_PID); videoModes = filterVideoModes(baseCamera.enumerateVideoModes()); } From 4bebc3d063bb053bc1d22bdad1e71aea69d07415 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Tue, 26 Nov 2019 01:45:35 -0500 Subject: [PATCH 72/84] Added method to set camera gain for PS3Eye --- .../vision/camera/CameraCapture.java | 9 ++++++++- .../vision/camera/USBCameraCapture.java | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java b/Main/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java index 2233eb2cc..152bccaa0 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java @@ -13,7 +13,7 @@ public interface CameraCapture extends ImageCapture { void setExposure(int exposure); /** - * Set the exposure of the camera + * Set the brightness of the camera * @param brightness the new brightness to set the camera to */ void setBrightness(int brightness); @@ -23,4 +23,11 @@ public interface CameraCapture extends ImageCapture { * @param mode the wanted mode */ void setVideoMode(VideoMode mode); + + /** + * Set the gain of the camera + * NOTE - Not all cameras support this. + * @param gain the new gain to set the camera to + */ + void setGain(int gain); } diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java index b3391bee8..d1f8371fb 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java @@ -42,7 +42,7 @@ public class USBCameraCapture implements CameraCapture { try { baseCamera.setExposureManual(exposure); } catch (VideoException e) { - System.err.println("Current camera does not support exposure change"); + System.err.println("Failed to change camera exposure!"); } } @@ -51,7 +51,7 @@ public class USBCameraCapture implements CameraCapture { try { baseCamera.setBrightness(brightness); } catch (VideoException e) { - System.err.println("Current camera does not support brightness change"); + System.err.println("Failed to change camera brightness!"); } } @@ -61,7 +61,19 @@ public class USBCameraCapture implements CameraCapture { baseCamera.setVideoMode(mode); properties.updateVideoMode(mode); } catch (VideoException e) { - System.err.println("Current camera does not support resolution change"); + System.err.println("Failed to change camera video mode!"); + } + } + + @Override + public void setGain(int gain) { + if (properties.isPS3Eye) { + try { + baseCamera.getProperty("gain_automatic").set(0); + baseCamera.getProperty("gain").set(gain); + } catch (Exception e) { + System.err.println("Failed to change camera gain!"); + } } } } From f6e56ce930fd35a9443098bcc39b478d1558437d Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Tue, 26 Nov 2019 02:55:05 -0500 Subject: [PATCH 73/84] Major pipeline optimizations, moved grabFrame in to vision thread --- .../chameleonvision/vision/VisionProcess.java | 131 +++--------------- .../vision/camera/USBCameraCapture.java | 5 +- .../vision/pipeline/CVPipeline2d.java | 3 +- .../vision/pipeline/CVPipelineResult.java | 5 + .../vision/pipeline/pipes/BlurPipe.java | 6 +- .../pipeline/pipes/Collect2dTargetsPipe.java | 48 ++++--- .../pipeline/pipes/Draw2dContoursPipe.java | 64 +++++---- .../pipeline/pipes/ErodeDilatePipe.java | 24 ++-- .../pipeline/pipes/FilterContoursPipe.java | 48 ++++--- .../pipeline/pipes/GroupContoursPipe.java | 85 ++++++------ .../vision/pipeline/pipes/HsvPipe.java | 6 +- .../vision/pipeline/pipes/RotateFlipPipe.java | 26 +++- .../pipeline/pipes/SortContoursPipe.java | 53 +++---- .../pipeline/pipes/SpeckleRejectPipe.java | 20 +-- .../java/com/chameleonvision/web/Server.java | 1 - 15 files changed, 236 insertions(+), 289 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index 30589dc71..5e5d62552 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -11,6 +11,7 @@ import com.google.gson.GsonBuilder; import edu.wpi.cscore.VideoMode; import edu.wpi.first.networktables.*; import edu.wpi.first.wpiutil.CircularBuffer; +import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; import java.util.ArrayList; @@ -21,7 +22,6 @@ public class VisionProcess { private final CameraCapture cameraCapture; private final List pipelines = new ArrayList<>(); - private final CameraFrameRunnable cameraRunnable; private final CameraStreamerRunnable streamRunnable; private final VisionProcessRunnable visionRunnable; public final CameraStreamer cameraStreamer; @@ -32,13 +32,10 @@ public class VisionProcess { private final CVPipelineSettings driverModeSettings = new CVPipelineSettings(); private CVPipeline driverModePipeline = new DriverVisionPipeline(driverModeSettings); - // shitty stuff - private volatile Mat lastCameraFrame = new Mat(); - private volatile boolean hasUnprocessedFrame = true; private volatile CVPipelineResult lastPipelineResult; // network table stuff - private NetworkTable defaultTable; + private final NetworkTable defaultTable; private NetworkTableEntry ntPipelineEntry; private NetworkTableEntry ntDriverModeEntry; private int ntDriveModeListenerID; @@ -57,14 +54,6 @@ public class VisionProcess { pipelines.add(new CVPipeline2d("New Pipeline")); setPipeline(0, false); - // Thread to grab frames from the camera - // TODO: (HIGH) fix video modes!!! - this.cameraRunnable = new CameraFrameRunnable(cameraCapture.getProperties().videoModes.get(0).fps); - - lastPipelineResult = new DriverVisionPipeline.DriverPipelineResult( - null, cameraRunnable.getFrame(new Mat()), 0 - ); - // Thread to put frames on the dashboard this.cameraStreamer = new CameraStreamer(cameraCapture, name); this.streamRunnable = new CameraStreamerRunnable(30, cameraStreamer); @@ -77,18 +66,8 @@ public class VisionProcess { } public void start() { - System.out.println("Starting NetworkTables"); + System.out.println("Starting NetworkTables."); initNT(defaultTable); - - System.out.println("Starting camera thread."); - new Thread(cameraRunnable).start(); - while (cameraRunnable.cameraFrame == null) { - try { - if (lastCameraFrame.cols() > 0) break; - } catch (Exception e) { -// e.printStackTrace(); - } - } System.out.println("Starting vision thread."); new Thread(visionRunnable).start(); System.out.println("Starting stream thread."); @@ -217,7 +196,7 @@ public class VisionProcess { //noinspection unchecked List targets = (List) data.targets; - ntTimeStampEntry.setDouble(data.processTime); + ntTimeStampEntry.setDouble(data.imageTimestamp); ntPitchEntry.setDouble(targets.get(0).pitch); ntYawEntry.setDouble(targets.get(0).yaw); ntAreaEntry.setDouble(targets.get(0).area); @@ -231,12 +210,12 @@ public class VisionProcess { ntYawEntry.setDouble(0.0); ntAreaEntry.setDouble(0.0); ntTimeStampEntry.setDouble(0.0); + ntAuxListEntry.setString(""); } } public void setVideoMode(VideoMode newMode) { cameraCapture.setVideoMode(newMode); - cameraRunnable.updateCameraFPS(newMode.fps); cameraStreamer.setNewVideoMode(newMode); } @@ -277,59 +256,6 @@ public class VisionProcess { return pipelines.get(pipelineIndex); } - /** - * CameraFrameRunnable grabs images from the cameraProcess - * at a specified loopTime - */ - protected class CameraFrameRunnable extends LoopingRunnable { - private Mat cameraFrame; - private long timestampMicros; - - private final Object frameLock = new Object(); - - /** - * CameraFrameRunnable grabs images from the cameraProcess - * at a specified framerate - * @param cameraFPS FPS of camera - */ - CameraFrameRunnable(int cameraFPS) { - // add 2 FPS to allow for a bit of overhead - // TODO: (low) test the effect of this - super(1000L/(cameraFPS + 2)); - } - - void updateCameraFPS(int newFPS) { - super.loopTimeMs = 1000L / (newFPS + 2); - } - - @Override - public void process() { - - // Grab camera frames - var camData = cameraCapture.getFrame(); - if (camData.getLeft().cols() > 0) { -// System.out.println("grabbing frame"); -// synchronized (frameLock) { -// cameraFrame = camData.getLeft(); -// } - timestampMicros = camData.getRight(); - camData.getLeft().copyTo(lastCameraFrame); - hasUnprocessedFrame = true; - - } - } - - public Mat getFrame(Mat dst) { - if (cameraFrame != null) { - dst = cameraFrame; - } else { - System.out.println("no frame"); - } - return dst; - } - - } - /** * VisionProcessRunnable will process images as quickly as possible */ @@ -337,48 +263,36 @@ public class VisionProcess { volatile Double fps = 0.0; private CircularBuffer fpsAveragingBuffer = new CircularBuffer(7); - @SuppressWarnings("FieldCanBeLocal") - private CVPipelineResult result; private Mat streamBuffer = new Mat(); @Override public void run() { var lastUpdateTimeNanos = System.nanoTime(); while(!Thread.interrupted()) { -// System.out.println("running vision process"); - while(!hasUnprocessedFrame) { - try { - Thread.sleep(3); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } + // blocking call, will block until camera has a new frame. + Pair camData = cameraCapture.getFrame(); - lastCameraFrame.copyTo(streamBuffer); // = //cameraRunnable.getFrame(streamBuffer); - hasUnprocessedFrame = false; - - if (streamBuffer.cols() > 0 && streamBuffer.rows() > 0) { - result = currentPipeline.runPipeline(streamBuffer); - lastPipelineResult = result; + Mat camFrame = camData.getLeft(); + if (camFrame.cols() > 0 && camFrame.rows() > 0) { + CVPipelineResult result = currentPipeline.runPipeline(camFrame); if (result != null) { + result.setTimestamp(camData.getRight()); + lastPipelineResult = result; updateNetworkTableData(lastPipelineResult); updateUI(lastPipelineResult); } - } var deltaTimeNanos = lastUpdateTimeNanos - System.nanoTime(); fpsAveragingBuffer.addFirst(1.0 / (deltaTimeNanos * 1E-09)); lastUpdateTimeNanos = System.nanoTime(); fps = getAverageFPS(); - - // TODO: (HIGH) do something with the result } } - public double getAverageFPS() { + double getAverageFPS() { var temp = 0.0; for(int i = 0; i < 7; i++) { temp += fpsAveragingBuffer.get(i); @@ -389,31 +303,32 @@ public class VisionProcess { } - private class CameraStreamerRunnable extends LoopingRunnable { + private static class CameraStreamerRunnable extends LoopingRunnable { - public final CameraStreamer streamer; + final CameraStreamer streamer; private Mat streamBuffer = new Mat(); private CameraStreamerRunnable(int cameraFPS, CameraStreamer streamer) { // add 2 FPS to allow for a bit of overhead // TODO: (low) test the effect of this - super(1000L/(cameraFPS + 2)); +// super(1000L/(cameraFPS + 2)); + super(10L); this.streamer = streamer; } @Override protected void process() { // System.out.println("running camera streamer"); - Mat latestMat = lastPipelineResult.outputMat; //visionRunnable.result; - if (latestMat != null && latestMat.cols() > 0) { - latestMat.copyTo(streamBuffer); - streamer.runStream(streamBuffer); - streamBuffer.release(); +// Mat latestMat = lastPipelineResult.outputMat; //visionRunnable.result; +// if (latestMat != null && latestMat.cols() > 0) { +// latestMat.copyTo(streamBuffer); +// streamer.runStream(streamBuffer); +// streamBuffer.release(); // if (toStreamMat != null && toStreamMat.cols() > 0) { // } else { // System.out.println("fuuuuck"); // } - } +// } } } } diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java index d1f8371fb..ff031de59 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java @@ -31,9 +31,8 @@ public class USBCameraCapture implements CameraCapture { @Override public Pair getFrame() { Long deltaTime; - synchronized (cvSink) { - deltaTime = cvSink.grabFrame(imageBuffer) * 1000L; - } + // TODO: Why multiply by 1000 here? + deltaTime = cvSink.grabFrame(imageBuffer) * 1000L; return Pair.of(imageBuffer, deltaTime); } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index 8bb2f3f51..898c9e57a 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -89,11 +89,10 @@ public class CVPipeline2d extends CVPipeline { public final boolean hasTarget; public final Mat outputMat; public final long processTime; + public long imageTimestamp = 0; public CVPipelineResult(List targets, Mat outputMat, long processTime) { this.targets = targets; @@ -16,4 +17,8 @@ public abstract class CVPipelineResult { this.outputMat = outputMat; this.processTime = processTime; } + + public void setTimestamp(long timestamp) { + imageTimestamp = timestamp; + } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java index 029e5e523..eeb175356 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java @@ -30,6 +30,7 @@ public class BlurPipe implements Pipe { try { Imgproc.blur(processBuffer, processBuffer, new Size(blurSize, blurSize)); processBuffer.copyTo(outputMat); + processBuffer.release(); } catch (CvException e) { System.err.println("(BlurPipe) Exception thrown by OpenCV: \n" + e.getMessage()); } @@ -37,10 +38,7 @@ public class BlurPipe implements Pipe { input.copyTo(outputMat); } - long processTime = System.nanoTime() - processStartNanos; - Pair output = Pair.of(outputMat, processTime); - processBuffer.release(); - return output; + return Pair.of(outputMat, processTime); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java index 87a627fc2..d1c0fb960 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java @@ -43,30 +43,32 @@ public class Collect2dTargetsPipe implements Pipe, List { - CVPipeline2d.Target2d t = new CVPipeline2d.Target2d(); - t.rawPoint = r; - switch (calibrationMode) { - case None: - t.calibratedX = camProps.centerX; - t.calibratedY = camProps.centerY; - break; - case Single: - t.calibratedX = calibrationPoint.get(0).doubleValue(); - t.calibratedY = calibrationPoint.get(1).doubleValue(); - break; - case Dual: - t.calibratedX = (r.center.y - calibrationB) / calibrationM; - t.calibratedY = (r.center.x * calibrationM) + calibrationB; - break; + if (input.size() > 0) { + for (RotatedRect r : input) { + CVPipeline2d.Target2d t = new CVPipeline2d.Target2d(); + t.rawPoint = r; + switch (calibrationMode) { + case None: + t.calibratedX = camProps.centerX; + t.calibratedY = camProps.centerY; + break; + case Single: + t.calibratedX = calibrationPoint.get(0).doubleValue(); + t.calibratedY = calibrationPoint.get(1).doubleValue(); + break; + case Dual: + t.calibratedX = (r.center.y - calibrationB) / calibrationM; + t.calibratedY = (r.center.x * calibrationM) + calibrationB; + break; + } + + t.pitch = calculatePitch(r.center.y, t.calibratedY); + t.yaw = calculateYaw(r.center.x, t.calibratedX); + t.area = r.size.area(); + + targets.add(t); } - - t.pitch = calculatePitch(r.center.y, t.calibratedY); - t.yaw = calculateYaw(r.center.x, t.calibratedX); - t.area = r.size.area(); - - targets.add(t); - }); + } long processTime = System.nanoTime() - processStartNanos; return Pair.of(targets, processTime); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java index 3fc361f47..69481fb68 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java @@ -17,6 +17,7 @@ public class Draw2dContoursPipe implements Pipe>, Ma private final Draw2dContoursSettings settings; private CaptureStaticProperties camProps; + private Mat processBuffer = new Mat(); private Mat outputMat = new Mat(); public Draw2dContoursPipe(Draw2dContoursSettings settings, CaptureStaticProperties camProps) { @@ -32,48 +33,51 @@ public class Draw2dContoursPipe implements Pipe>, Ma public Pair run(Pair> input) { long processStartNanos = System.nanoTime(); - outputMat = input.getLeft(); + if (settings.showCrosshair || settings.showCentroid || settings.showMaximumBox || settings.showRotatedBox) { + input.getLeft().copyTo(processBuffer); - if (input.getRight() != null && !input.getRight().isEmpty()) { - for (RotatedRect r : input.getRight()) { - if (r == null) continue; + if (input.getRight().size() > 0) { + for (RotatedRect r : input.getRight()) { + if (r == null) continue; - List drawnContour = new ArrayList<>(); - Point[] vertices = new Point[4]; - r.points(vertices); - MatOfPoint contour = new MatOfPoint(vertices); - drawnContour.add(contour); + List drawnContour = new ArrayList<>(); + Point[] vertices = new Point[4]; + r.points(vertices); + MatOfPoint contour = new MatOfPoint(vertices); + drawnContour.add(contour); - if (settings.showCentroid) { - Imgproc.circle(outputMat, r.center, 3, Helpers.colorToScalar(settings.centroidColor)); - } + if (settings.showCentroid) { + Imgproc.circle(processBuffer, r.center, 3, Helpers.colorToScalar(settings.centroidColor)); + } - if (settings.showRotatedBox) { - Imgproc.drawContours(outputMat, drawnContour, 0, Helpers.colorToScalar(settings.rotatedBoxColor), settings.boxOutlineSize); - } + if (settings.showRotatedBox) { + Imgproc.drawContours(processBuffer, drawnContour, 0, Helpers.colorToScalar(settings.rotatedBoxColor), settings.boxOutlineSize); + } - if (settings.showMaximumBox) { - Rect box = Imgproc.boundingRect(contour); - Imgproc.rectangle(outputMat, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), Helpers.colorToScalar(settings.maximumBoxColor), settings.boxOutlineSize); + if (settings.showMaximumBox) { + Rect box = Imgproc.boundingRect(contour); + Imgproc.rectangle(processBuffer, new Point(box.x, box.y), new Point((box.x + box.width), (box.y + box.height)), Helpers.colorToScalar(settings.maximumBoxColor), settings.boxOutlineSize); + } } } - } - if (settings.showCrosshair) { - Point xMax = new Point(camProps.centerX + 10, camProps.centerY); - Point xMin = new Point(camProps.centerX - 10, camProps.centerY); - Point yMax = new Point(camProps.centerX, camProps.centerY + 10); - Point yMin = new Point(camProps.centerX, camProps.centerY - 10); - if (outputMat != null && outputMat.cols() > 0) { - Imgproc.line(outputMat, xMax, xMin, Helpers.colorToScalar(settings.crosshairColor), 2); - Imgproc.line(outputMat, yMax, yMin, Helpers.colorToScalar(settings.crosshairColor), 2); + if (settings.showCrosshair) { + Point xMax = new Point(camProps.centerX + 10, camProps.centerY); + Point xMin = new Point(camProps.centerX - 10, camProps.centerY); + Point yMax = new Point(camProps.centerX, camProps.centerY + 10); + Point yMin = new Point(camProps.centerX, camProps.centerY - 10); + Imgproc.line(processBuffer, xMax, xMin, Helpers.colorToScalar(settings.crosshairColor), 2); + Imgproc.line(processBuffer, yMax, yMin, Helpers.colorToScalar(settings.crosshairColor), 2); } + + processBuffer.copyTo(outputMat); + processBuffer.release(); + } else { + input.getLeft().copyTo(outputMat); } long processTime = System.nanoTime() - processStartNanos; - Pair output = Pair.of(outputMat, processTime); - outputMat.release(); - return output; + return Pair.of(outputMat, processTime); } public static class Draw2dContoursSettings { diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java index 4534f8ae1..3798856d5 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java @@ -30,20 +30,24 @@ public class ErodeDilatePipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - input.copyTo(processBuffer); + if (erode || dilate) { + input.copyTo(processBuffer); - if (erode) { - Imgproc.erode(processBuffer, processBuffer, kernel); - } + if (erode) { + Imgproc.erode(processBuffer, processBuffer, kernel); + } - if (dilate) { - Imgproc.erode(processBuffer, processBuffer, kernel); + if (dilate) { + Imgproc.erode(processBuffer, processBuffer, kernel); + } + + processBuffer.copyTo(outputMat); + processBuffer.release(); + } else { + input.copyTo(outputMat); } long processTime = System.nanoTime() - processStartNanos; - processBuffer.copyTo(outputMat); - Pair output = Pair.of(outputMat, processTime); - processBuffer.release(); - return output; + return Pair.of(outputMat, processTime); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java index 9870e43a5..121216214 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java @@ -40,30 +40,32 @@ public class FilterContoursPipe implements Pipe, List maxArea) { - continue; + if (input.size() > 0) { + for (MatOfPoint Contour : input) { + try { + double contourArea = Imgproc.contourArea(Contour); + double AreaRatio = (contourArea / camProps.imageArea) * 100; + double minArea = (MathHandler.sigmoid(area.get(0))); + double maxArea = (MathHandler.sigmoid(area.get(1))); + if (AreaRatio < minArea || AreaRatio > maxArea) { + continue; + } + var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); + double minExtent = (extent.get(0).doubleValue() * rect.size.area()) / 100; + double maxExtent = (extent.get(1).doubleValue() * rect.size.area()) / 100; + if (contourArea <= minExtent || contourArea >= maxExtent) { + continue; + } + Rect bb = Imgproc.boundingRect(Contour); + double aspectRatio = ((double)bb.width / bb.height); + if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { + continue; + } + filteredContours.add(Contour); + } catch (Exception e) { + System.err.println("Error while filtering contours"); + e.printStackTrace(); } - var rect = Imgproc.minAreaRect(new MatOfPoint2f(Contour.toArray())); - double minExtent = (extent.get(0).doubleValue() * rect.size.area()) / 100; - double maxExtent = (extent.get(1).doubleValue() * rect.size.area()) / 100; - if (contourArea <= minExtent || contourArea >= maxExtent) { - continue; - } - Rect bb = Imgproc.boundingRect(Contour); - double aspectRatio = ((double)bb.width / bb.height); - if (aspectRatio < ratio.get(0).doubleValue() || aspectRatio > ratio.get(1).doubleValue()) { - continue; - } - filteredContours.add(Contour); - } catch (Exception e) { - System.err.println("Error while filtering contours"); - e.printStackTrace(); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java index 070a895a8..b2d3df9aa 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java @@ -41,56 +41,59 @@ public class GroupContoursPipe implements Pipe, List sorted = new ArrayList<>(input); - sorted.sort(sortByMomentsX); + if (input.size() > 0) { - Collections.reverse(sorted); + List sorted = new ArrayList<>(input); + sorted.sort(sortByMomentsX); - switch (group) { - case Single: { - input.forEach(c -> { - MatOfPoint2f contour = new MatOfPoint2f(); - contour.fromArray(c.toArray()); - if (contour.cols() != 0 && contour.rows() != 0) { - RotatedRect rect = Imgproc.minAreaRect(contour); - groupedContours.add(rect); - } - }); - break; - } - case Dual: { - for (var i = 0; i < input.size(); i++) { - List finalContourList = new ArrayList<>(input.get(i).toList()); - - try { - MatOfPoint firstContour = input.get(i); - MatOfPoint secondContour = input.get(i + 1); - - if (isIntersecting(firstContour, secondContour)) { - finalContourList.addAll(secondContour.toList()); - } else { - finalContourList.clear(); - continue; - } - - intersectMatA.release(); - intersectMatB.release(); - firstContour.release(); - secondContour.release(); + Collections.reverse(sorted); + switch (group) { + case Single: { + input.forEach(c -> { MatOfPoint2f contour = new MatOfPoint2f(); - contour.fromList(finalContourList); - + contour.fromArray(c.toArray()); if (contour.cols() != 0 && contour.rows() != 0) { RotatedRect rect = Imgproc.minAreaRect(contour); groupedContours.add(rect); } - } catch (IndexOutOfBoundsException e) { - System.err.println("GroupContours: WTF"); - finalContourList.clear(); - } + }); + break; + } + case Dual: { + for (var i = 0; i < input.size(); i++) { + List finalContourList = new ArrayList<>(input.get(i).toList()); + + try { + MatOfPoint firstContour = input.get(i); + MatOfPoint secondContour = input.get(i + 1); + + if (isIntersecting(firstContour, secondContour)) { + finalContourList.addAll(secondContour.toList()); + } else { + finalContourList.clear(); + continue; + } + + intersectMatA.release(); + intersectMatB.release(); + firstContour.release(); + secondContour.release(); + + MatOfPoint2f contour = new MatOfPoint2f(); + contour.fromList(finalContourList); + + if (contour.cols() != 0 && contour.rows() != 0) { + RotatedRect rect = Imgproc.minAreaRect(contour); + groupedContours.add(rect); + } + } catch (IndexOutOfBoundsException e) { + System.err.println("GroupContours: WTF"); + finalContourList.clear(); + } + } + break; } - break; } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java index 454f851d5..47544297b 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/HsvPipe.java @@ -38,11 +38,11 @@ public class HsvPipe implements Pipe { System.err.println("(HsvPipe) Exception thrown by OpenCV: \n" + e.getMessage()); } - long processTime = System.nanoTime() - processStartNanos; processBuffer.copyTo(outputMat); - Pair output = Pair.of(outputMat, processTime); processBuffer.release(); - return output; + + long processTime = System.nanoTime() - processStartNanos; + return Pair.of(outputMat, processTime); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java index a4a604dfb..94cf483ff 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/RotateFlipPipe.java @@ -28,15 +28,27 @@ public class RotateFlipPipe implements Pipe { public Pair run(Mat input) { long processStartNanos = System.nanoTime(); - input.copyTo(processBuffer); + boolean shouldFlip = !flip.equals(ImageFlipMode.NONE); + boolean shouldRotate = !rotation.equals(ImageRotation.DEG_0); - Core.flip(processBuffer, processBuffer, flip.value); - Core.rotate(processBuffer, processBuffer, rotation.value); + if (shouldFlip || shouldRotate) { + input.copyTo(processBuffer); + + if (shouldFlip) { + Core.flip(processBuffer, processBuffer, flip.value); + } + + if (shouldRotate) { + Core.rotate(processBuffer, processBuffer, rotation.value); + } + + processBuffer.copyTo(outputMat); + processBuffer.release(); + } else { + input.copyTo(outputMat); + } long processTime = System.nanoTime() - processStartNanos; - processBuffer.copyTo(outputMat); - Pair output = Pair.of(outputMat, processTime); - processBuffer.release(); - return output; + return Pair.of(outputMat, processTime); } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java index 329b848a2..7281f2a6c 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java @@ -44,32 +44,35 @@ public class SortContoursPipe implements Pipe, List 0) { + sortedContours.addAll(input); + + switch (sort) { + case Largest: + sortedContours.sort(SortByLargestComparator); + break; + case Smallest: + sortedContours.sort(SortBySmallestComparator); + break; + case Highest: + sortedContours.sort(SortByHighestComparator); + break; + case Lowest: + sortedContours.sort(SortByLowestComparator); + break; + case Leftmost: + sortedContours.sort(SortByLeftmostComparator); + break; + case Rightmost: + sortedContours.sort(SortByRightmostComparator); + break; + case Centermost: + sortedContours.sort(SortByCentermostComparator); + break; + default: + break; + } } long processTime = System.nanoTime() - processStartNanos; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java index 7edebf7f7..058e955d6 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java @@ -27,19 +27,21 @@ public class SpeckleRejectPipe implements Pipe, List 0) { + double averageArea = 0.0; - for (MatOfPoint c : input) { - averageArea += Imgproc.contourArea(c); - } + for (MatOfPoint c : input) { + averageArea += Imgproc.contourArea(c); + } - averageArea /= input.size(); + averageArea /= input.size(); - double minAllowedArea = minPercentOfAvg / 100.0 * averageArea; + double minAllowedArea = minPercentOfAvg / 100.0 * averageArea; - for (MatOfPoint c : input) { - if (Imgproc.contourArea(c) >= minAllowedArea) { - despeckledContours.add(c); + for (MatOfPoint c : input) { + if (Imgproc.contourArea(c) >= minAllowedArea) { + despeckledContours.add(c); + } } } diff --git a/Main/src/main/java/com/chameleonvision/web/Server.java b/Main/src/main/java/com/chameleonvision/web/Server.java index 6141dc173..bbb1c0461 100644 --- a/Main/src/main/java/com/chameleonvision/web/Server.java +++ b/Main/src/main/java/com/chameleonvision/web/Server.java @@ -20,7 +20,6 @@ public class Server { ws.onClose(ctx -> { handler.onClose(ctx); System.out.println("Socket Disconnected"); - // TODO: (HIGH) add generalSettingsSave ConfigManager.saveSettings(); }); ws.onBinaryMessage(ctx -> { From 6506281ddf497b140303bf364ce17111d2bacc84 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Tue, 26 Nov 2019 17:56:21 -0500 Subject: [PATCH 74/84] Fix Jetson WPILib native dep --- Main/chameleon-vision.iml | 15 +-------------- Main/pom.xml | 6 ++++++ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 5cbae1d91..69873dfae 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -12,20 +12,6 @@ - - - - - - - - - - - - - - @@ -69,6 +55,7 @@ + diff --git a/Main/pom.xml b/Main/pom.xml index 9dab6dc06..7af954887 100644 --- a/Main/pom.xml +++ b/Main/pom.xml @@ -203,6 +203,12 @@ 2020.1.1-beta-2-98-gb058dcf linuxx86-64 + + edu.wpi.first.ntcore + ntcore-jni + 2020.1.1-beta-2-98-gb058dcf + linuxaarch64bionic + edu.wpi.first.ntcore ntcore-jni From d588b1a69ea9ad00f3895ab1f9bf2dd38f5f4334 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Tue, 26 Nov 2019 17:57:51 -0500 Subject: [PATCH 75/84] Camera configuration rewrite, begin on CameraStreamer threading fix --- .../chameleonvision/config/CameraConfig.java | 182 +++++++++++++++--- .../config/CameraJsonConfig.java | 37 ++++ .../chameleonvision/config/ConfigManager.java | 114 +++-------- .../config/FullCameraConfiguration.java | 18 ++ .../{FileHelper.java => JacksonHelper.java} | 27 +-- .../chameleonvision/vision/VisionManager.java | 36 ++-- .../chameleonvision/vision/VisionProcess.java | 40 ++-- .../vision/camera/USBCameraCapture.java | 4 +- .../vision/camera/USBCameraProperties.java | 4 +- .../vision/pipeline/CVPipeline.java | 2 +- .../java/com/chameleonvision/web/Server.java | 2 +- 11 files changed, 293 insertions(+), 173 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/config/CameraJsonConfig.java create mode 100644 Main/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java rename Main/src/main/java/com/chameleonvision/util/{FileHelper.java => JacksonHelper.java} (61%) diff --git a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java index b416eb65d..f002223f7 100644 --- a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java +++ b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java @@ -1,37 +1,167 @@ package com.chameleonvision.config; -import com.chameleonvision.vision.camera.USBCameraCapture; -import com.chameleonvision.vision.camera.USBCameraProperties; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.chameleonvision.util.JacksonHelper; +import com.chameleonvision.vision.pipeline.CVPipelineSettings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; public class CameraConfig { - public final double fov; - public final String path; - public final String name; - public final String nickname; - @JsonCreator - public CameraConfig( - @JsonProperty("fov") double fov, - @JsonProperty("path") String path, - @JsonProperty("name") String name, - @JsonProperty("nickname") String nickname) { - this.fov = fov; - this.path = path; - this.name = name; - this.nickname = nickname; + private static final Path camerasConfigFolderPath = Paths.get(ConfigManager.SettingsPath.toString(), "cameras"); + + private final String cameraConfigName; + private final CameraJsonConfig preliminaryConfig; + + CameraConfig(CameraJsonConfig config) { + preliminaryConfig = config; + cameraConfigName = preliminaryConfig.name.replace(' ', '_'); } - public CameraConfig(String path, String name) { - this.fov = USBCameraProperties.DEFAULT_FOV; - this.path = path; - this.name = name; - this.nickname = name; + public CameraJsonConfig load() { + checkFolder(); + checkConfig(); + checkPipelines(); + checkDriverMode(); + + // todo: add pipelines and drivermode loads + return loadConfig(); } - public static CameraConfig fromUSBCameraProcess(USBCameraCapture process) { - USBCameraProperties camProps = process.getProperties(); - return new CameraConfig(camProps.FOV, camProps.name, camProps.path, camProps.getNickname()); + private CameraJsonConfig loadConfig() { + CameraJsonConfig config = preliminaryConfig; + try { + config = JacksonHelper.deserializer(getConfigPath(), CameraJsonConfig.class); + } catch (IOException e) { + System.err.printf("Failed to load camera config: %s - using default.\n", getConfigPath().toString()); + } + return config; + } + + List loadPipelines() { + List pipelines = new ArrayList<>(); + try { + var pipelineArray = JacksonHelper.deserializer(getPipelinesPath(), CVPipelineSettings[].class); + if (pipelineArray != null) { + pipelines = Arrays.asList(pipelineArray); + } + } catch (IOException e) { + System.err.println("Failed to load camera pipelines: " + getPipelinesPath().toString()); + } + return pipelines; + } + + CVPipelineSettings loadDriverMode() { + CVPipelineSettings driverMode = new CVPipelineSettings(); + driverMode.nickname = "DRIVERMODE"; + try { + driverMode = JacksonHelper.deserializer(getDriverModePath(), CVPipelineSettings.class); + } catch (IOException e) { + System.err.println("Failed to load camera drivermode: " + getDriverModePath().toString()); + } + return driverMode; + } + + void saveConfig(CameraJsonConfig config) { + try { + JacksonHelper.serializer(getConfigPath(), config); + } catch (IOException e) { + System.err.println("Failed to save camera config file: " + getConfigPath().toString()); + } + } + + void savePipelines(List pipelines) { + try { + JacksonHelper.serializer(getPipelinesPath(), pipelines); + } catch (IOException e) { + System.err.println("Failed to save camera pipelines file: " + getConfigPath().toString()); + } + } + + void saveDriverMode(CVPipelineSettings driverMode) { + try { + JacksonHelper.serializer(getDriverModePath(), driverMode); + } catch (IOException e) { + System.err.println("Failed to save camera drivermode file: " + getDriverModePath().toString()); + } + } + + private void checkFolder() { + if (!folderExists()) { + try { + Files.createDirectory(getFolderPath()); + } catch (IOException e) { + System.err.println("Failed to create camera config folder: " + getFolderPath().toString()); + } + } + } + + private void checkConfig() { + if (!configExists()) { + try { + JacksonHelper.serializer(getConfigPath(), preliminaryConfig); + } catch (IOException e) { + System.err.println("Failed to create camera config file: " + getConfigPath().toString()); + } + } + } + + private void checkPipelines() { + if (!pipelinesExists()) { + try { + Files.createFile(getPipelinesPath()); + } catch (IOException e) { + System.err.println("Failed to create camera pipelines file: " + getPipelinesPath().toString()); + } + } + } + + private void checkDriverMode() { + if (!driverModeExists()) { + try { + CVPipelineSettings newDriverModeSettings = new CVPipelineSettings(); + newDriverModeSettings.nickname = "DRIVERMODE"; + JacksonHelper.serializer(getDriverModePath(), newDriverModeSettings); + } catch (IOException e) { + System.err.println("Failed to create camera drivermode file: " + getDriverModePath().toString()); + } + } + } + + private Path getFolderPath() { + return Paths.get(camerasConfigFolderPath.toString(), cameraConfigName); + } + + private Path getConfigPath() { + return Paths.get(getFolderPath().toString(), "camera.json"); + } + + private Path getPipelinesPath() { + return Paths.get(getFolderPath().toString(), "pipelines.json"); + } + + private Path getDriverModePath() { + return Paths.get(getFolderPath().toString(), "drivermode.json"); + } + + private boolean folderExists() { + return Files.exists(getFolderPath()); + } + + private boolean configExists() { + return folderExists() && Files.exists(getConfigPath()); + } + + private boolean pipelinesExists() { + return folderExists() && Files.exists(getPipelinesPath()); + } + + private boolean driverModeExists() { + return folderExists() && Files.exists(getDriverModePath()); } } diff --git a/Main/src/main/java/com/chameleonvision/config/CameraJsonConfig.java b/Main/src/main/java/com/chameleonvision/config/CameraJsonConfig.java new file mode 100644 index 000000000..ebd9e6b56 --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/config/CameraJsonConfig.java @@ -0,0 +1,37 @@ +package com.chameleonvision.config; + +import com.chameleonvision.vision.camera.USBCameraCapture; +import com.chameleonvision.vision.camera.USBCameraProperties; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class CameraJsonConfig { + public final double fov; + public final String path; + public final String name; + public final String nickname; + + @JsonCreator + public CameraJsonConfig( + @JsonProperty("fov") double fov, + @JsonProperty("path") String path, + @JsonProperty("name") String name, + @JsonProperty("nickname") String nickname) { + this.fov = fov; + this.path = path; + this.name = name; + this.nickname = nickname; + } + + public CameraJsonConfig(String path, String name) { + this.fov = USBCameraProperties.DEFAULT_FOV; + this.path = path; + this.name = name; + this.nickname = name; + } + + public static CameraJsonConfig fromUSBCameraProcess(USBCameraCapture process) { + USBCameraProperties camProps = process.getProperties(); + return new CameraJsonConfig(camProps.FOV, camProps.name, camProps.path, camProps.getNickname()); + } +} diff --git a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java index 68212fc78..9c8329911 100644 --- a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java +++ b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java @@ -1,7 +1,7 @@ package com.chameleonvision.config; import com.chameleonvision.util.ProgramDirectoryUtilities; -import com.chameleonvision.util.FileHelper; +import com.chameleonvision.util.JacksonHelper; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import java.io.File; @@ -10,15 +10,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; public class ConfigManager { private ConfigManager() {} - private static final Path SettingsPath = Paths.get(ProgramDirectoryUtilities.getProgramDirectory(), "settings"); - private static final Path cameraConfigPath = Paths.get(SettingsPath.toString(), "cameras"); + static final Path SettingsPath = Paths.get(ProgramDirectoryUtilities.getProgramDirectory(), "settings"); private static final Path settingsFilePath = Paths.get(SettingsPath.toString(), "settings.json"); + private static final LinkedHashMap cameraConfigs = new LinkedHashMap<>(); + public static GeneralSettings settings = new GeneralSettings(); private static boolean settingsFolderExists() { return Files.exists(SettingsPath); } @@ -38,13 +40,13 @@ public class ConfigManager { boolean settingsFileEmpty = settingsFileExists() && new File(settingsFilePath.toString()).length() == 0; if (settingsFileEmpty || !settingsFileExists()) { try { - FileHelper.Serializer(settingsFilePath, settings); + JacksonHelper.serializer(settingsFilePath, settings); } catch (IOException e) { e.printStackTrace(); } } else { try { - settings = FileHelper.DeSerializer(settingsFilePath, GeneralSettings.class); + settings = JacksonHelper.deserializer(settingsFilePath, GeneralSettings.class); } catch (IOException e) { System.err.println("Failed to load settings.json, using defaults."); } @@ -52,13 +54,14 @@ public class ConfigManager { } public static void initializeSettings() { + System.out.println("Settings folder: " + SettingsPath.toString()); checkSettingsFolder(); checkSettingsFile(); } private static void saveSettingsFile() { try { - FileHelper.Serializer(settingsFilePath, settings); + JacksonHelper.serializer(settingsFilePath, settings); } catch (IOException e) { System.err.println("Failed to save settings.json!"); } @@ -69,97 +72,36 @@ public class ConfigManager { saveSettingsFile(); } - private static Path getCameraSpecificFolderPath(String cameraName) { - return Paths.get(cameraConfigPath.toString(), cameraName); - } - - private static Path getCameraSpecificConfigPath(String cameraName) { - return Paths.get(getCameraSpecificFolderPath(cameraName).toString(), "camera.json"); - } - - private static Path getCameraSpecificPipelinesPath(String cameraName) { - return Paths.get(getCameraSpecificFolderPath(cameraName).toString(), "pipelines.json"); - } - - private static Path getCameraSpecificDriverModePath(String cameraName) { - return Paths.get(getCameraSpecificFolderPath(cameraName).toString(), "drivermode.json"); - } - - private static boolean cameraFolderExists(String cameraName) { - return Files.exists(getCameraSpecificFolderPath(cameraName)); - } - - private static boolean cameraConfigExists(String cameraName) { - return cameraFolderExists(cameraName) && Files.exists(getCameraSpecificConfigPath(cameraName)); - } - - private static boolean cameraPipelinesExists(String cameraName) { - return cameraFolderExists(cameraName) && Files.exists(getCameraSpecificPipelinesPath(cameraName)); - } - - private static boolean cameraDriverModeExists(String cameraName) { - return cameraFolderExists(cameraName) && Files.exists(getCameraSpecificDriverModePath(cameraName)); - } - // TODO: (HIGH) cleanup! - public static List initializeCameraConfig(List preliminaryConfigs) { - var configList = new ArrayList(); + public static List initializeCameras(List preliminaryConfigs) { + List configList = new ArrayList<>(); checkSettingsFolder(); // loop over all the camera names and try to create settings folders for it - preliminaryConfigs.forEach((preliminaryConfig) -> { - String cameraName = preliminaryConfig.name; + for (CameraJsonConfig preliminaryConfig : preliminaryConfigs) { + CameraConfig cameraConfiguration = new CameraConfig(preliminaryConfig); + cameraConfigs.put(preliminaryConfig.name, cameraConfiguration); - final Path cameraConfigFolderPath = getCameraSpecificFolderPath(cameraName); - final Path cameraConfigPath = getCameraSpecificConfigPath(cameraName); + CameraJsonConfig camJsonConfig = cameraConfiguration.load(); + List pipelines = cameraConfiguration.loadPipelines(); + CVPipelineSettings driverMode = cameraConfiguration.loadDriverMode(); - // check if the config folder exists, and if not, create it - if (!cameraFolderExists(cameraName)) { - try { - Files.createDirectory(cameraConfigFolderPath); - } catch (IOException e) { - System.err.println("Failed to create camera config folder!"); - } - } else { - CameraConfig config = preliminaryConfig; - - // check if the config exists, and if not, create it - if(!cameraConfigExists(cameraName)) { - try { - FileHelper.Serializer(cameraConfigPath, preliminaryConfig); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - try { - config = FileHelper.DeSerializer(cameraConfigPath, CameraConfig.class); - } catch (IOException e) { - e.printStackTrace(); - } - } - configList.add(config); - } - }); + configList.add(new FullCameraConfiguration(camJsonConfig, pipelines, driverMode)); + } return configList; } - public static void saveCameraPipelines(String cameraName, List pipelines) throws IOException { - Path cameraFolder = Paths.get(cameraConfigPath.toString(), cameraName); - Path filePath = Paths.get(cameraFolder.toString(), cameraName,"pipelines.json"); - FileHelper.CheckPath(cameraFolder); - FileHelper.Serializer(filePath, pipelines); + + public static void saveCameraConfig(String cameraName, CameraJsonConfig config) { + cameraConfigs.get(cameraName).saveConfig(config); } - public static void saveCameraDriverMode(String cameraName, CVPipelineSettings driverMode) throws IOException { - Path cameraFolder = Paths.get(cameraConfigPath.toString(), cameraName); - Path filePath = Paths.get(cameraFolder.toString(), cameraName,"driverMode.json"); - FileHelper.CheckPath(cameraFolder); - FileHelper.Serializer(filePath, driverMode); + + public static void saveCameraPipelines(String cameraName, List pipelines) { + cameraConfigs.get(cameraName).savePipelines(pipelines); } - public static void saveCameraConfig(String cameraName, CameraConfig config) throws IOException { - Path cameraFolder = Paths.get(cameraConfigPath.toString(), cameraName); - Path filePath = Paths.get(cameraFolder.toString(), cameraName,"driverMode.json"); - FileHelper.CheckPath(cameraFolder); - FileHelper.Serializer(filePath, config); + + public static void saveCameraDriverMode(String cameraName, CVPipelineSettings driverMode) { + cameraConfigs.get(cameraName).saveDriverMode(driverMode); } } diff --git a/Main/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java b/Main/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java new file mode 100644 index 000000000..208e007fd --- /dev/null +++ b/Main/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java @@ -0,0 +1,18 @@ +package com.chameleonvision.config; + +import com.chameleonvision.vision.pipeline.CVPipelineSettings; + +import java.util.List; + +public class FullCameraConfiguration { + public final CameraJsonConfig cameraConfig; + public final List pipelines; + public final CVPipelineSettings drivermode; + + + public FullCameraConfiguration(CameraJsonConfig cameraConfig, List pipelines, CVPipelineSettings drivermode) { + this.cameraConfig = cameraConfig; + this.pipelines = pipelines; + this.drivermode = drivermode; + } +} diff --git a/Main/src/main/java/com/chameleonvision/util/FileHelper.java b/Main/src/main/java/com/chameleonvision/util/JacksonHelper.java similarity index 61% rename from Main/src/main/java/com/chameleonvision/util/FileHelper.java rename to Main/src/main/java/com/chameleonvision/util/JacksonHelper.java index cec221a36..68dfd437a 100644 --- a/Main/src/main/java/com/chameleonvision/util/FileHelper.java +++ b/Main/src/main/java/com/chameleonvision/util/JacksonHelper.java @@ -9,38 +9,21 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -public class FileHelper { - private FileHelper() {} // no construction, utility class +public class JacksonHelper { + private JacksonHelper() {} // no construction, utility class - public static void CheckPath(String path) { - if (path.equals("")) return; - Path realPath = Path.of(path); - CheckPath(realPath); - } - - public static void CheckPath(Path path) { - if (!Files.exists(path)) { - try { - Files.createDirectories(path); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - public static void Serializer(Path path, Object object) throws IOException { + public static void serializer(Path path, Object object) throws IOException { PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder().allowIfBaseType(Object.class).build(); ObjectMapper objectMapper = JsonMapper.builder().activateDefaultTyping(ptv, ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT).build(); objectMapper.writerWithDefaultPrettyPrinter().writeValue(new File(path.toString()), object); } - public static T DeSerializer(Path path, Class ref) throws IOException { + public static T deserializer(Path path, Class ref) throws IOException { PolymorphicTypeValidator ptv = BasicPolymorphicTypeValidator.builder().allowIfBaseType(ref).build(); ObjectMapper objectMapper = JsonMapper.builder().activateDefaultTyping(ptv).build(); File jsonFile = new File(path.toString()); if (jsonFile.exists() && jsonFile.length() > 0) { - T readObject = objectMapper.readValue(jsonFile, ref); - return readObject; + return objectMapper.readValue(jsonFile, ref); } return null; } diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java index b0b148796..628e3b38d 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java @@ -1,7 +1,8 @@ package com.chameleonvision.vision; -import com.chameleonvision.config.CameraConfig; +import com.chameleonvision.config.CameraJsonConfig; import com.chameleonvision.config.ConfigManager; +import com.chameleonvision.config.FullCameraConfiguration; import com.chameleonvision.util.Helpers; import com.chameleonvision.util.Platform; import com.chameleonvision.vision.camera.CameraCapture; @@ -20,7 +21,7 @@ public class VisionManager { } private static final LinkedHashMap usbCameraInfosByCameraName = new LinkedHashMap<>(); - private static final LinkedList loadedCameraConfigs = new LinkedList<>(); + private static final LinkedList loadedCameraConfigs = new LinkedList<>(); private static final LinkedList visionProcesses = new LinkedList<>(); private static class VisionProcessManageable { @@ -57,7 +58,7 @@ public class VisionManager { } // load the config - List preliminaryConfigs = new ArrayList<>(); + List preliminaryConfigs = new ArrayList<>(); usbCameraInfosByCameraName.values().forEach((cameraInfo) -> { String truePath; @@ -68,10 +69,10 @@ public class VisionManager { truePath = Arrays.stream(cameraInfo.otherPaths).filter(x -> x.contains("/dev/v4l/by-path")).findFirst().orElse(cameraInfo.path); } - preliminaryConfigs.add(new CameraConfig(truePath, cameraInfo.name)); + preliminaryConfigs.add(new CameraJsonConfig(truePath, cameraInfo.name)); }); - loadedCameraConfigs.addAll(ConfigManager.initializeCameraConfig(preliminaryConfigs)); + loadedCameraConfigs.addAll(ConfigManager.initializeCameras(preliminaryConfigs)); // TODO: (HIGH) Load pipelines from json // UsbCameraInfosByCameraName.forEach((cameraName, cameraInfo) -> { @@ -85,10 +86,15 @@ public class VisionManager { public static boolean initializeProcesses() { for (int i = 0; i < loadedCameraConfigs.size(); i++) { - CameraConfig config = loadedCameraConfigs.get(i); - CameraCapture camera = new USBCameraCapture(config); - VisionProcess process = new VisionProcess(camera, config.name); - visionProcesses.add(new VisionProcessManageable(i, config.name, process)); + FullCameraConfiguration config = loadedCameraConfigs.get(i); + + CameraJsonConfig cameraJsonConfig = config.cameraConfig; + + CameraCapture camera = new USBCameraCapture(cameraJsonConfig); + VisionProcess process = new VisionProcess(camera, cameraJsonConfig.name); + config.pipelines.forEach(process::addPipeline); + process.setDriverModeSettings(config.drivermode); + visionProcesses.add(new VisionProcessManageable(i, cameraJsonConfig.name, process)); } currentUIVisionProcess = getVisionProcessByIndex(0); return true; @@ -136,14 +142,10 @@ public class VisionManager { String cameraName = process.getCamera().getProperties().name; List pipelines = process.getPipelines().stream().map(cvPipeline -> cvPipeline.settings).collect(Collectors.toList()); CVPipelineSettings driverMode = process.getDriverModeSettings(); - CameraConfig config = CameraConfig.fromUSBCameraProcess((USBCameraCapture) process.getCamera()); - try { - ConfigManager.saveCameraPipelines(cameraName, pipelines); - ConfigManager.saveCameraDriverMode(cameraName, driverMode); - ConfigManager.saveCameraConfig(cameraName, config); - } catch (IOException e) { - e.printStackTrace(); - } + CameraJsonConfig config = CameraJsonConfig.fromUSBCameraProcess((USBCameraCapture) process.getCamera()); + ConfigManager.saveCameraPipelines(cameraName, pipelines); + ConfigManager.saveCameraDriverMode(cameraName, driverMode); + ConfigManager.saveCameraConfig(cameraName, config); }); } diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index 5e5d62552..f974221f8 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -17,6 +17,8 @@ import org.opencv.core.Mat; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; public class VisionProcess { @@ -29,11 +31,12 @@ public class VisionProcess { private CVPipeline currentPipeline; private int currentPipelineIndex = 0; - private final CVPipelineSettings driverModeSettings = new CVPipelineSettings(); - private CVPipeline driverModePipeline = new DriverVisionPipeline(driverModeSettings); + private CVPipeline driverModePipeline = new DriverVisionPipeline(new CVPipelineSettings()); private volatile CVPipelineResult lastPipelineResult; + BlockingQueue streamFrameQueue = new LinkedBlockingDeque<>(1); + // network table stuff private final NetworkTable defaultTable; private NetworkTableEntry ntPipelineEntry; @@ -240,6 +243,12 @@ public class VisionProcess { pipelines.add(pipeline); } + public void addPipeline(CVPipelineSettings settings) { + if (settings instanceof CVPipeline2dSettings) { + pipelines.add(new CVPipeline2d((CVPipeline2dSettings) settings)); + } + } + public CameraCapture getCamera() { return cameraCapture; } @@ -248,6 +257,11 @@ public class VisionProcess { return (currentPipeline == driverModePipeline); } + public void setDriverModeSettings(CVPipelineSettings settings) { + + driverModePipeline.settings = settings; + } + public CVPipelineSettings getDriverModeSettings() { return driverModePipeline.settings; } @@ -285,6 +299,12 @@ public class VisionProcess { } } + try { + streamFrameQueue.add(lastPipelineResult.outputMat); + } catch (Exception e) { + System.out.println("Vision running faster than stream"); + } + var deltaTimeNanos = lastUpdateTimeNanos - System.nanoTime(); fpsAveragingBuffer.addFirst(1.0 / (deltaTimeNanos * 1E-09)); lastUpdateTimeNanos = System.nanoTime(); @@ -310,25 +330,13 @@ public class VisionProcess { private CameraStreamerRunnable(int cameraFPS, CameraStreamer streamer) { // add 2 FPS to allow for a bit of overhead - // TODO: (low) test the effect of this -// super(1000L/(cameraFPS + 2)); - super(10L); + super(1000L/(cameraFPS + 2)); this.streamer = streamer; } @Override protected void process() { -// System.out.println("running camera streamer"); -// Mat latestMat = lastPipelineResult.outputMat; //visionRunnable.result; -// if (latestMat != null && latestMat.cols() > 0) { -// latestMat.copyTo(streamBuffer); -// streamer.runStream(streamBuffer); -// streamBuffer.release(); -// if (toStreamMat != null && toStreamMat.cols() > 0) { -// } else { -// System.out.println("fuuuuck"); -// } -// } + } } } diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java index ff031de59..d7912b428 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java @@ -1,6 +1,6 @@ package com.chameleonvision.vision.camera; -import com.chameleonvision.config.CameraConfig; +import com.chameleonvision.config.CameraJsonConfig; import edu.wpi.cscore.CvSink; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.VideoException; @@ -15,7 +15,7 @@ public class USBCameraCapture implements CameraCapture { private Mat imageBuffer = new Mat(); private USBCameraProperties properties; - public USBCameraCapture(CameraConfig config) { + public USBCameraCapture(CameraJsonConfig config) { baseCamera = new UsbCamera(config.name, config.path); cvSink = CameraServer.getInstance().getVideo(baseCamera); properties = new USBCameraProperties(baseCamera, config); diff --git a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java index 56fd9521c..d98e221ba 100644 --- a/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java +++ b/Main/src/main/java/com/chameleonvision/vision/camera/USBCameraProperties.java @@ -1,6 +1,6 @@ package com.chameleonvision.vision.camera; -import com.chameleonvision.config.CameraConfig; +import com.chameleonvision.config.CameraJsonConfig; import com.chameleonvision.util.Platform; import com.chameleonvision.vision.image.CaptureProperties; import edu.wpi.cscore.UsbCamera; @@ -40,7 +40,7 @@ public class USBCameraProperties extends CaptureProperties { private String nickname; public double FOV; - USBCameraProperties(UsbCamera baseCamera, CameraConfig config) { + USBCameraProperties(UsbCamera baseCamera, CameraJsonConfig config) { FOV = config.fov; name = config.name; path = config.path; diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java index 9b27f65d2..52970d204 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java @@ -10,7 +10,7 @@ import org.opencv.core.Mat; public abstract class CVPipeline { protected Mat outputMat = new Mat(); CameraCapture cameraCapture; - public final S settings; + public S settings; protected CVPipeline(S settings) { this.settings = settings; diff --git a/Main/src/main/java/com/chameleonvision/web/Server.java b/Main/src/main/java/com/chameleonvision/web/Server.java index bbb1c0461..7f8fa656f 100644 --- a/Main/src/main/java/com/chameleonvision/web/Server.java +++ b/Main/src/main/java/com/chameleonvision/web/Server.java @@ -10,7 +10,7 @@ public class Server { public static void main(int port) { handler = new ServerHandler(); - Javalin app = Javalin.create(javalinConfig -> javalinConfig.showJavalinBanner=false); + Javalin app = Javalin.create(javalinConfig -> javalinConfig.showJavalinBanner = false); app.config.addStaticFiles("web"); app.ws("/websocket", ws -> { ws.onConnect(ctx -> { From 8b8138eb502999b345e8dbe3448908ede0be755c Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Tue, 26 Nov 2019 23:03:07 -0500 Subject: [PATCH 76/84] Config tweaks, fixed CameraStreamer --- .../chameleonvision/config/ConfigManager.java | 2 +- .../chameleonvision/vision/VisionManager.java | 21 +++++++++++++ .../chameleonvision/vision/VisionProcess.java | 18 ++++++++--- .../vision/pipeline/CVPipeline2d.java | 31 +++++++------------ .../vision/pipeline/CVPipelineResult.java | 4 +-- .../pipeline/pipes/ErodeDilatePipe.java | 2 +- .../vision/pipeline/pipes/OutputMatPipe.java | 26 +++++++++++++--- .../java/com/chameleonvision/web/Server.java | 2 +- .../chameleonvision/web/ServerHandler.java | 25 ++++++--------- 9 files changed, 82 insertions(+), 49 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java index 9c8329911..0802a1194 100644 --- a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java +++ b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java @@ -67,7 +67,7 @@ public class ConfigManager { } } - public static void saveSettings() { + public static void saveGeneralSettings() { checkSettingsFolder(); saveSettingsFile(); } diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java index 628e3b38d..e102b697b 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionManager.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionManager.java @@ -7,6 +7,7 @@ import com.chameleonvision.util.Helpers; import com.chameleonvision.util.Platform; import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.vision.camera.USBCameraCapture; +import com.chameleonvision.vision.pipeline.CVPipeline; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.UsbCameraInfo; @@ -136,6 +137,7 @@ public class VisionManager { return currentUIVisionProcess.getPipelines().stream().map(cvPipeline -> cvPipeline.settings.nickname).collect(Collectors.toList()); } + public static void saveCameras() { visionProcesses.forEach((vpm) -> { VisionProcess process = vpm.visionProcess; @@ -149,6 +151,25 @@ public class VisionManager { }); } + private static String getCurrentCameraName() { + return currentUIVisionProcess.getCamera().getProperties().name; + } + + public static void saveCurrentCameraSettings() { + CameraJsonConfig config = CameraJsonConfig.fromUSBCameraProcess((USBCameraCapture) currentUIVisionProcess.getCamera()); + ConfigManager.saveCameraConfig(getCurrentCameraName(), config); + } + + public static void saveCurrentCameraPipelines() { + List pipelineSettings = currentUIVisionProcess.getPipelines().stream().map(pipeline -> pipeline.settings).collect(Collectors.toList()); + ConfigManager.saveCameraPipelines(getCurrentCameraName(), pipelineSettings); + } + + public static void saveCurrentCameraDriverMode() { + CVPipelineSettings driverModeSettings = currentUIVisionProcess.getDriverModeSettings(); + ConfigManager.saveCameraDriverMode(getCurrentCameraName(), driverModeSettings); + } + public static List getCurrentCameraResolutionList() { return currentUIVisionProcess.getCamera().getProperties().getVideoModes().stream().map(Helpers::VideoModeToString).collect(Collectors.toList()); } diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index f974221f8..fca45771d 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -35,7 +35,7 @@ public class VisionProcess { private volatile CVPipelineResult lastPipelineResult; - BlockingQueue streamFrameQueue = new LinkedBlockingDeque<>(1); + private BlockingQueue streamFrameQueue = new LinkedBlockingDeque<>(1); // network table stuff private final NetworkTable defaultTable; @@ -323,10 +323,9 @@ public class VisionProcess { } - private static class CameraStreamerRunnable extends LoopingRunnable { + private class CameraStreamerRunnable extends LoopingRunnable { final CameraStreamer streamer; - private Mat streamBuffer = new Mat(); private CameraStreamerRunnable(int cameraFPS, CameraStreamer streamer) { // add 2 FPS to allow for a bit of overhead @@ -336,7 +335,18 @@ public class VisionProcess { @Override protected void process() { - + try { + if (!streamFrameQueue.isEmpty()) { + Mat latestMat = streamFrameQueue.take(); + if (!latestMat.empty()) { + streamer.runStream(latestMat); + } else { + System.out.println("stream mat empty"); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } } } } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java index 898c9e57a..906354328 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java @@ -16,7 +16,6 @@ import static com.chameleonvision.vision.pipeline.CVPipeline2d.*; public class CVPipeline2d extends CVPipeline { private Mat rawCameraMat = new Mat(); - private Mat hsvOutputMat = new Mat(); private RotateFlipPipe rotateFlipPipe; private BlurPipe blurPipe; @@ -32,13 +31,10 @@ public class CVPipeline2d extends CVPipeline hsvResult = hsvPipe.run(erodeDilateResult.getLeft()); totalPipelineTimeNanos += hsvResult.getRight(); - // Todo: move to a pipe - try { - Imgproc.cvtColor(hsvResult.getLeft(), hsvOutputMat, Imgproc.COLOR_GRAY2BGR, 3); - } catch (CvException e) { - System.err.println("(CVPipeline2d) Exception thrown by OpenCV: \n" + e.getMessage()); - } - Pair, Long> findContoursResult = findContoursPipe.run(hsvResult.getLeft()); totalPipelineTimeNanos += findContoursResult.getRight(); @@ -153,8 +142,8 @@ public class CVPipeline2d extends CVPipeline, Long> collect2dTargetsResult = collect2dTargetsPipe.run(sortContoursResult.getLeft()); totalPipelineTimeNanos += collect2dTargetsResult.getRight(); - // takes pair of (Mat of original camera image, Mat of HSV thresholded image) - Pair outputMatResult = outputMatPipe.run(Pair.of(rawCameraMat, hsvOutputMat)); + // takes pair of (Mat of original camera image (8UC3), Mat of HSV thresholded image(8UC1)) + Pair outputMatResult = outputMatPipe.run(Pair.of(rawCameraMat, hsvResult.getLeft())); totalPipelineTimeNanos += outputMatResult.getRight(); // takes pair of (Mat to draw on, List of sorted contours) @@ -175,13 +164,15 @@ public class CVPipeline2d extends CVPipeline { public final List targets; public final boolean hasTarget; - public final Mat outputMat; + public final Mat outputMat = new Mat(); public final long processTime; public long imageTimestamp = 0; public CVPipelineResult(List targets, Mat outputMat, long processTime) { this.targets = targets; hasTarget = targets != null && !targets.isEmpty(); - this.outputMat = outputMat; + outputMat.copyTo(this.outputMat); this.processTime = processTime; } diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java index 3798856d5..215e3ecdd 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java @@ -38,7 +38,7 @@ public class ErodeDilatePipe implements Pipe { } if (dilate) { - Imgproc.erode(processBuffer, processBuffer, kernel); + Imgproc.dilate(processBuffer, processBuffer, kernel); } processBuffer.copyTo(outputMat); diff --git a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java index df7972108..82d02228b 100644 --- a/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java +++ b/Main/src/main/java/com/chameleonvision/vision/pipeline/pipes/OutputMatPipe.java @@ -1,12 +1,15 @@ package com.chameleonvision.vision.pipeline.pipes; import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.CvException; import org.opencv.core.Mat; +import org.opencv.imgproc.Imgproc; public class OutputMatPipe implements Pipe, Mat> { private boolean showThresholded; + private Mat processBuffer = new Mat(); private Mat outputMat = new Mat(); public OutputMatPipe(boolean showThresholded) { @@ -17,15 +20,30 @@ public class OutputMatPipe implements Pipe, Mat> { this.showThresholded = showThresholded; } + /** + * + * @param input Input object for pipe + * Left is raw camera mat (8UC3), Right is HSV threshold mat (8UC1) + * @return Returns desired output Mat, and processing time in nanoseconds + */ @Override public Pair run(Pair input) { long processStartNanos = System.nanoTime(); - outputMat = showThresholded ? input.getRight() : input.getLeft(); + if (showThresholded) { + try { + input.getRight().copyTo(processBuffer); + Imgproc.cvtColor(processBuffer, processBuffer, Imgproc.COLOR_GRAY2BGR, 3); + processBuffer.copyTo(outputMat); + processBuffer.release(); + } catch (CvException e) { + System.err.println("(OutputMat) Exception thrown by OpenCV: \n" + e.getMessage()); + } + } else { + input.getLeft().copyTo(outputMat); + } long processTime = System.nanoTime() - processStartNanos; - Pair output = Pair.of(outputMat, processTime); - outputMat.release(); - return output; + return Pair.of(outputMat, processTime); } } diff --git a/Main/src/main/java/com/chameleonvision/web/Server.java b/Main/src/main/java/com/chameleonvision/web/Server.java index 7f8fa656f..121b78634 100644 --- a/Main/src/main/java/com/chameleonvision/web/Server.java +++ b/Main/src/main/java/com/chameleonvision/web/Server.java @@ -20,7 +20,7 @@ public class Server { ws.onClose(ctx -> { handler.onClose(ctx); System.out.println("Socket Disconnected"); - ConfigManager.saveSettings(); + ConfigManager.saveGeneralSettings(); }); ws.onBinaryMessage(ctx -> { handler.onBinaryMessage(ctx); diff --git a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java index b3608b02d..ee4e984e5 100644 --- a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java +++ b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java @@ -58,7 +58,7 @@ public class ServerHandler { for (HashMap.Entry e : data.entrySet()) { setField(ConfigManager.settings, e.getKey(), e.getValue()); } - ConfigManager.saveSettings(); + ConfigManager.saveGeneralSettings(); sendFullSettings(); break; } @@ -68,7 +68,7 @@ public class ServerHandler { currentProcess.getDriverModeSettings().brightness = (Integer) data.get("brightness"); currentProcess.setDriverMode((Boolean) data.get("isDriver")); - VisionManager.saveCameras(); + VisionManager.saveCurrentCameraDriverMode(); break; } case "cameraSettings": { @@ -90,20 +90,20 @@ public class ServerHandler { // currentCamera.getProperties().setCamVideoMode(newResolution, true); // } - VisionManager.saveCameras(); + VisionManager.saveCurrentCameraSettings(); sendFullSettings(); break; } case "changeCameraName": { currentCamera.getProperties().setNickname((String) entry.getValue()); sendFullSettings(); - ConfigManager.saveSettings(); + VisionManager.saveCurrentCameraSettings(); break; } case "changePipelineName": { currentPipeline.settings.nickname = ((String) entry.getValue()); sendFullSettings(); - ConfigManager.saveSettings(); + VisionManager.saveCurrentCameraPipelines(); break; } case "duplicatePipeline": { @@ -121,7 +121,7 @@ public class ServerHandler { } else { currentProcess.addPipeline(origPipeline); } - ConfigManager.saveSettings(); + VisionManager.saveCurrentCameraPipelines(); break; } case "command": { @@ -129,7 +129,7 @@ public class ServerHandler { case "addNewPipeline": currentProcess.addPipeline(); sendFullSettings(); - ConfigManager.saveSettings(); + VisionManager.saveCurrentCameraPipelines(); break; // TODO: (HIGH) this never worked before, re-visit now that VisionProcess is written sanely case "deleteCurrentPipeline": @@ -143,10 +143,10 @@ public class ServerHandler { // cam.deletePipeline(); // cam.setCurrentPipelineIndex(nextIndex); // sendFullSettings(); -// ConfigManager.saveSettings(); +// VisionManager.saveCurrentCameraPipelines(); break; case "save": - ConfigManager.saveSettings(); + ConfigManager.saveGeneralSettings(); System.out.println("saved Settings"); break; } @@ -161,17 +161,10 @@ public class ServerHandler { case "currentPipeline": { currentProcess.setPipeline((Integer) entry.getValue(), true); sendFullSettings(); - try { - currentCamera.setBrightness((int) currentPipeline.settings.brightness); - currentCamera.setExposure((int) currentPipeline.settings.exposure); - } catch (Exception e) { - continue; - } break; } default: { setField(currentPipeline.settings, entry.getKey(), entry.getValue()); - switch (entry.getKey()) { case "exposure": { currentCamera.setExposure((Integer) entry.getValue()); From b3f39f02018e61707afc6ac79315c7e9832f68b9 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 26 Nov 2019 22:18:29 -0800 Subject: [PATCH 77/84] Only post the first (up to) 5 targets --- .../main/java/com/chameleonvision/vision/VisionProcess.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index fca45771d..0e8f19419 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; +import static java.lang.Math.min; + public class VisionProcess { private final CameraCapture cameraCapture; @@ -203,7 +205,7 @@ public class VisionProcess { ntPitchEntry.setDouble(targets.get(0).pitch); ntYawEntry.setDouble(targets.get(0).yaw); ntAreaEntry.setDouble(targets.get(0).area); - ntAuxListEntry.setString(gson.toJson(targets)); + ntAuxListEntry.setString(gson.toJson(targets.subList(0, min(targets.size(), 5)))); } else if (data instanceof CVPipeline3d.CVPipeline3dResult) { // TODO: (2.1) 3d stuff... From 019ad14c73e08b6156bb6952447f10e71d813cb4 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 26 Nov 2019 22:26:53 -0800 Subject: [PATCH 78/84] use mkdirs() method to create the directory --- Main/src/main/java/com/chameleonvision/config/CameraConfig.java | 2 ++ .../src/main/java/com/chameleonvision/config/ConfigManager.java | 1 + 2 files changed, 3 insertions(+) diff --git a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java index f002223f7..aec7e21f3 100644 --- a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java +++ b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java @@ -3,6 +3,7 @@ package com.chameleonvision.config; import com.chameleonvision.util.JacksonHelper; import com.chameleonvision.vision.pipeline.CVPipelineSettings; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -94,6 +95,7 @@ public class CameraConfig { private void checkFolder() { if (!folderExists()) { try { + new File(getFolderPath().toUri()).mkdirs(); Files.createDirectory(getFolderPath()); } catch (IOException e) { System.err.println("Failed to create camera config folder: " + getFolderPath().toString()); diff --git a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java index 0802a1194..a5e442986 100644 --- a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java +++ b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java @@ -29,6 +29,7 @@ public class ConfigManager { private static void checkSettingsFolder() { if (!settingsFolderExists()) { try { + new File(SettingsPath.toUri()).mkdirs(); Files.createDirectory(SettingsPath); } catch (IOException e) { e.printStackTrace(); From ebf87c8b2ab7e3245ca8e12ba9c48477f10922e3 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Wed, 27 Nov 2019 01:37:50 -0500 Subject: [PATCH 79/84] Fix for cameras folder not getting created --- .../main/java/com/chameleonvision/config/CameraConfig.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java index aec7e21f3..5d0d5fbd3 100644 --- a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java +++ b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java @@ -94,10 +94,7 @@ public class CameraConfig { private void checkFolder() { if (!folderExists()) { - try { - new File(getFolderPath().toUri()).mkdirs(); - Files.createDirectory(getFolderPath()); - } catch (IOException e) { + if (!(new File(getFolderPath().toUri()).mkdirs())) { System.err.println("Failed to create camera config folder: " + getFolderPath().toString()); } } From ca7f15aeb74bc358381c6b718df2434b65509378 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 27 Nov 2019 08:58:20 -0800 Subject: [PATCH 80/84] Catch file exceptions in CameraConfig and ConfigManager --- .../java/com/chameleonvision/config/CameraConfig.java | 10 ++++++++-- .../java/com/chameleonvision/config/ConfigManager.java | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java index 5d0d5fbd3..57d37d559 100644 --- a/Main/src/main/java/com/chameleonvision/config/CameraConfig.java +++ b/Main/src/main/java/com/chameleonvision/config/CameraConfig.java @@ -4,6 +4,7 @@ import com.chameleonvision.util.JacksonHelper; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -94,8 +95,13 @@ public class CameraConfig { private void checkFolder() { if (!folderExists()) { - if (!(new File(getFolderPath().toUri()).mkdirs())) { - System.err.println("Failed to create camera config folder: " + getFolderPath().toString()); + try { + if (!(new File(getFolderPath().toUri()).mkdirs())) { + System.err.println("Failed to create camera config folder: " + getFolderPath().toString()); + } + } catch(Exception e) { + if(!(e instanceof java.nio.file.FileAlreadyExistsException || e instanceof java.nio.file.FileAlreadyExistsException)) + System.err.println("Failed to create camera config folder: " + getFolderPath().toString()); } } } diff --git a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java index a5e442986..4aff1fb18 100644 --- a/Main/src/main/java/com/chameleonvision/config/ConfigManager.java +++ b/Main/src/main/java/com/chameleonvision/config/ConfigManager.java @@ -32,7 +32,8 @@ public class ConfigManager { new File(SettingsPath.toUri()).mkdirs(); Files.createDirectory(SettingsPath); } catch (IOException e) { - e.printStackTrace(); + if(!(e instanceof java.nio.file.FileAlreadyExistsException || e instanceof java.nio.file.FileAlreadyExistsException)) + e.printStackTrace(); } } } From e195cce026de671dd4fd35ee40cb2664e1ec3f6d Mon Sep 17 00:00:00 2001 From: ori agranat Date: Wed, 27 Nov 2019 21:46:07 +0200 Subject: [PATCH 81/84] bug fixes in ui and bug fixes in sever handler -- updated pom --- Main/pom.xml | 6 --- .../chameleonvision/vision/VisionProcess.java | 15 +++--- .../chameleonvision/web/ServerHandler.java | 50 ++----------------- chameleon-client/src/store.js | 2 +- .../src/views/CameraViewes/InputTab.vue | 2 +- 5 files changed, 16 insertions(+), 59 deletions(-) diff --git a/Main/pom.xml b/Main/pom.xml index 7af954887..be09c112a 100644 --- a/Main/pom.xml +++ b/Main/pom.xml @@ -85,12 +85,6 @@ commons-math3 3.6.1 - - - com.google.code.gson - gson - 2.8.5 - org.msgpack msgpack-core diff --git a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java index 0e8f19419..d8ccd5aa4 100644 --- a/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/Main/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -6,8 +6,8 @@ import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.vision.camera.CameraStreamer; import com.chameleonvision.vision.pipeline.*; import com.chameleonvision.web.ServerHandler; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import edu.wpi.cscore.VideoMode; import edu.wpi.first.networktables.*; import edu.wpi.first.wpiutil.CircularBuffer; @@ -20,7 +20,6 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; -import static java.lang.Math.min; public class VisionProcess { @@ -51,7 +50,7 @@ public class VisionProcess { private NetworkTableEntry ntAreaEntry; private NetworkTableEntry ntTimeStampEntry; private NetworkTableEntry ntValidEntry; - private Gson gson = new GsonBuilder().setPrettyPrinting().create(); + private ObjectMapper objectMapper; VisionProcess(CameraCapture cameraCapture, String name) { this.cameraCapture = cameraCapture; @@ -172,6 +171,7 @@ public class VisionProcess { center.add(bestTarget.rawPoint.center.y); calculated.put("pitch", bestTarget.pitch); calculated.put("yaw", bestTarget.yaw); + calculated.put("area", bestTarget.area); } else if (data instanceof CVPipeline3d.CVPipeline3dResult) { // TODO: (2.1) 3d stuff in UI } else { @@ -205,8 +205,11 @@ public class VisionProcess { ntPitchEntry.setDouble(targets.get(0).pitch); ntYawEntry.setDouble(targets.get(0).yaw); ntAreaEntry.setDouble(targets.get(0).area); - ntAuxListEntry.setString(gson.toJson(targets.subList(0, min(targets.size(), 5)))); - + try { + ntAreaEntry.setString(objectMapper.writeValueAsString(targets)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } } else if (data instanceof CVPipeline3d.CVPipeline3dResult) { // TODO: (2.1) 3d stuff... } diff --git a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java index ee4e984e5..71aea9519 100644 --- a/Main/src/main/java/com/chameleonvision/web/ServerHandler.java +++ b/Main/src/main/java/com/chameleonvision/web/ServerHandler.java @@ -214,9 +214,9 @@ public class ServerHandler { broadcastMessage(obj, null);//Broadcasts the message to every user } - private static HashMap getOrdinalPipeline() throws IllegalAccessException { + private static HashMap getOrdinalPipeline(Class cvClass) throws IllegalAccessException { HashMap tmp = new HashMap<>(); - for (Field field : CVPipelineSettings.class.getFields()) { // iterate over every field in CVPipelineSettings + for (Field field :cvClass.getFields()) { // iterate over every field in CVPipelineSettings try { if (!field.getType().isEnum()) { // if the field is not an enum, get it based on the current pipeline tmp.put(field.getName(), field.get(VisionManager.getCurrentUIVisionProcess().getCurrentPipeline().settings)); @@ -263,45 +263,6 @@ public class ServerHandler { return tmp; } - private static Map settingsToMap(GeneralSettings settings) { - Map map = new HashMap<>(); - map.put("team_number", settings.teamNumber); - map.put("connection_type", settings.connectionType); - map.put("ip", settings.ip); - map.put("gateway", settings.gateway); - map.put("netmask", settings.netmask); - map.put("hostname", settings.hostname); - map.put("curr_camera", settings.currentCamera); - map.put("curr_pipeline", settings.currentPipeline); - - return map; - } - - private static Map pipelineToMap(CVPipelineSettings s) { - Map map = new HashMap<>(); - map.put("exposure", s.exposure); - map.put("brightness", s.brightness); - if(s instanceof CVPipeline2dSettings) { - var s_ = (CVPipeline2dSettings) s; - map.put("orientation", s.flipMode.name()); - map.put("hue", s_.hue); - map.put("saturation", s_.saturation); - map.put("value", s_.value); - map.put("erode", s_.erode); - map.put("dilate", s_.dilate); - map.put("area", s_.area); - map.put("ratio", s_.ratio); - map.put("extent", s_.extent); - map.put("is_binary", s_.isBinary); - map.put("sort_mode", s_.sortMode.name()); - map.put("target_group", s_.targetGroup.name()); - map.put("target_intersection", s_.targetIntersection.name()); - map.put("M", s_.dualTargetCalibrationM); - map.put("B", s_.dualTargetCalibrationB); - map.put("is_calibrated", !s_.calibrationMode.equals(CalibrationMode.None)); - } - return map; - } public static void sendFullSettings() { //General settings Map fullSettings = new HashMap<>(); @@ -311,16 +272,15 @@ public class ServerHandler { CVPipeline currentPipeline = currentProcess.getCurrentPipeline(); try { - fullSettings.putAll(settingsToMap(ConfigManager.settings)); - fullSettings.putAll(pipelineToMap(currentPipeline.settings)); +// fullSettings.putAll(settingsToMap(ConfigManager.settings)); +// fullSettings.putAll(pipelineToMap(currentPipeline.settings)); fullSettings.put("settings", getOrdinalSettings()); fullSettings.put("cameraSettings", getOrdinalCameraSettings()); fullSettings.put("cameraList", VisionManager.getAllCameraNicknames()); - fullSettings.put("pipeline", getOrdinalPipeline()); + fullSettings.put("pipeline", getOrdinalPipeline(currentPipeline.settings.getClass())); fullSettings.put("driverMode", getOrdinalDriver()); fullSettings.put("pipelineList", VisionManager.getCurrentCameraPipelineNicknames()); fullSettings.put("resolutionList", VisionManager.getCurrentCameraResolutionList()); - fullSettings.put("FOV", currentCamera.getProperties().FOV); fullSettings.put("port", currentProcess.cameraStreamer.getStreamPort()); fullSettings.put("currentPipelineIndex", VisionManager.getCurrentUIVisionProcess().getCurrentPipelineIndex()); fullSettings.put("currentCameraIndex", VisionManager.getCurrentUIVisionProcessIndex()); diff --git a/chameleon-client/src/store.js b/chameleon-client/src/store.js index b0c9fbd2d..c4353706b 100644 --- a/chameleon-client/src/store.js +++ b/chameleon-client/src/store.js @@ -20,7 +20,7 @@ export default new Vuex.Store({ pipeline: { exposure: 0, brightness: 0, - orientation: 0, + flipMode: 0, hue: [0, 15], saturation: [0, 15], value: [0, 25], diff --git a/chameleon-client/src/views/CameraViewes/InputTab.vue b/chameleon-client/src/views/CameraViewes/InputTab.vue index 4f1681739..c872e0109 100644 --- a/chameleon-client/src/views/CameraViewes/InputTab.vue +++ b/chameleon-client/src/views/CameraViewes/InputTab.vue @@ -2,7 +2,7 @@
-
From 6b5bd75dbe17edecd757108219c085b57723ffe5 Mon Sep 17 00:00:00 2001 From: Banks Troutman Date: Wed, 27 Nov 2019 14:59:36 -0500 Subject: [PATCH 82/84] Config fixes, add JUnit maven dependency, potential fix for stream latency --- Main/chameleon-vision.iml | 19 +++++++++++----- Main/pom.xml | 18 ++++++++++----- .../main/java/com/chameleonvision/Debug.java | 19 ++++++++++++++++ .../main/java/com/chameleonvision/Main.java | 6 ++--- .../chameleonvision/config/CameraConfig.java | 1 - .../chameleonvision/config/ConfigManager.java | 16 +++++++++----- .../chameleonvision/vision/VisionManager.java | 2 +- .../chameleonvision/vision/VisionProcess.java | 22 ++++++++----------- ...geRotation.java => ImageRotationMode.java} | 4 ++-- .../vision/enums/Orientation.java | 7 ------ .../vision/pipeline/CVPipeline2d.java | 14 ++++++------ .../vision/pipeline/pipes/RotateFlipPipe.java | 10 ++++----- .../pipeline/pipes/SortContoursPipe.java | 10 +++++---- .../chameleonvision/web/ServerHandler.java | 5 ++++- .../config/ConfigManagerTest.java | 19 ++++++++++++++++ 15 files changed, 110 insertions(+), 62 deletions(-) create mode 100644 Main/src/main/java/com/chameleonvision/Debug.java rename Main/src/main/java/com/chameleonvision/vision/enums/{ImageRotation.java => ImageRotationMode.java} (80%) delete mode 100644 Main/src/main/java/com/chameleonvision/vision/enums/Orientation.java create mode 100644 Main/src/test/java/com/chameleonvision/config/ConfigManagerTest.java diff --git a/Main/chameleon-vision.iml b/Main/chameleon-vision.iml index 69873dfae..a0d6af26a 100644 --- a/Main/chameleon-vision.iml +++ b/Main/chameleon-vision.iml @@ -6,6 +6,7 @@ + @@ -44,6 +45,12 @@ + + + + + + @@ -58,11 +65,11 @@ - - - - - - + + + + + + \ No newline at end of file diff --git a/Main/pom.xml b/Main/pom.xml index be09c112a..76effa8c3 100644 --- a/Main/pom.xml +++ b/Main/pom.xml @@ -119,6 +119,12 @@ 2.10.1 + + org.junit.jupiter + junit-jupiter-engine + 5.5.2 + +