diff --git a/.gitignore b/.gitignore index 12b3f0298..3cccab404 100644 --- a/.gitignore +++ b/.gitignore @@ -113,8 +113,10 @@ chameleon-server/.classpath chameleon-server/.project chameleon-server/settings chameleon-server/dependency-reduced-pom.xml +chameleon-server/chameleon-vision.iml New client/chameleon-client/* *.prefs *.jfr +.DS_Store diff --git a/README.md b/README.md index 881a043b2..c467fba49 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + # Chameleon-Vision [![CircleCI](https://img.shields.io/circleci/build/github/Chameleon-Vision/chameleon-vision/dev?label=dev&logo=name)](https://circleci.com/gh/Chameleon-Vision/workflows/chameleon-vision/tree/dev) @@ -5,62 +7,69 @@ Chameleon Vision is free open-source software for FRC teams to use for vision proccesing on their robots. -## Getting started -See Deployment for notes on how to deploy the project on a live system. +There instructions are for compiling (contributing) and running the source-code of the project. +This is NOT intended for the co-processor setup or your testing PC. +To run the program normally (from a build .jar file), take a look at our ReadTheDocs documentation for installation [here](https://chameleon-vision.readthedocs.io/en/latest/installation/coprocessor-setup.html) -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. -(Coming soon!) + +These instruction are for the Chameleon Vision's backend/server in Java + +To run the UI's sourcecode (optional) see the UI's [readme](https://github.com/Chameleon-Vision/chameleon-vision/blob/master/chameleon-client/README.md) + +## Hardware +Currently any 64-Bit devices (Windows, Linux and Mac OS) are supported. +32 Bit devices are not supported. + +At least one USB camera ([supported](https://chameleon-vision.readthedocs.io/en/latest/hardware/supported-hardware.html#supported-cameras) one is recommended) + +## Development setup ### Prerequisites ---- -#### For the co-processor -- Java 12 Runtime (See Deployment - Software) -- Avahi Daemon +- Java Development Kit 12: +Follow the correct instructions for your platform from [BellSoft](https://bell-sw.com/pages/liberica_install_guide-12.0.2/) +- Chameleon-vision source code +Clone via a git client or download as zip and extract the source code into a empty folder +#### For the co-processor(Linux system) +- Avahi Daemon: +`sudo apt-get install avahi-daemon avahi-discover avahi-utils libnss-mdns mdns-scan` #### For the driver station - Bonjour - - -## Deployment -Deploying is as simple as uploading the chameleon-vision-1.xx.jar file to your target device. -Run the program with `java -jar chameleon-vision-1.xx.jar` - -## Software - -### Java 12 -Follow the correct instructions for your platform from [BellSoft](https://bell-sw.com/pages/liberica_install_guide-12.0.2/) - -### Avahi Daemon (Linux systems) -`sudo apt-get install avahi-daemon avahi-discover avahi-utils libnss-mdns mdns-scan` - -### Bonjour (Windows Systems) Download and install Bonjour [from here](https://support.apple.com/kb/DL999?locale=en_US) +- VC++ Redistributable (Windows only) +Download and install [this](https://aka.ms/vs/16/release/vc_redist.x64.exe) -## Hardware +## Importing to IDEA +We recommend the use of [Intellij Idea](https://www.jetbrains.com/idea/) for running the source-code -### ARM Co-processors -Currently only Raspberry Pi 3 or 4 models with at least 1GB of RAM are tested and supported. -Additional ARM-based single board computers (Odroid, Nvidia Jetson, etc.) will be supported in the near future. +1. Import Project +2. Choose the path to `chameleon-server` inside the copy of Chameleon-Vision that you cloned or downloaded -### x86 Computers -Currently any 64-Bit devices (Windows, Linux and Mac OS) are supported. -32 Bit devices are not supported. +![](https://i.vgy.me/KmrzCV.png) +3. Import the project as a `Maven` project + +![](https://i.vgy.me/2ltb7B.png) + +4. Under `JDK for importer` choose the JDK 12 you downloaded earlier +5. Maven will automatically download the necessary dependencies +6. Run `Main` under `src/main/java/com/chameleonvision/` + ## Authors * **Sagi Frimer** - *initial work* - websocket, settings manager, UI -* **Ori Agranat** - *main coder* - vision loop, UI, websocket, networktables +* **Ori Agranat** - *main coder* - project manager, vision loop, UI, websocket, networktables -* **Omer Zipory** - *developer* - vision loop, websocket, networking +* **Omer Zipory** - *developer* - vision loop, websocket, networking, documentation, UI -* **Banks Troutman** - *developer* - vision loop, websocket, networking +* **Banks Troutman** - *developer* - vision loop, websocket, networking, project structue -* **Matt Morley** - *developer* - documentation +* **Matt Morley** - *developer* - vision loop, project structue, documentation, solvePNP ## Acknowledgments @@ -71,8 +80,6 @@ Currently any 64-Bit devices (Windows, Linux and Mac OS) are supported. * [Javalin](https://javalin.io/) -* [Spring Framework](https://spring.io/) - * [JSON](https://json.org) * [Google](https://github.com/google) - Specifically [Gson](https://github.com/google/gson) diff --git a/chameleon-client/README.md b/chameleon-client/README.md index d3a694552..36cff5a2e 100644 --- a/chameleon-client/README.md +++ b/chameleon-client/README.md @@ -1,11 +1,18 @@ -# chameleon-client +# Chameleon Client UI + +## Install Node.js + +Follow [this](https://nodejs.org/en/) link ## Project setup +Run this one time, this command downloades the packages the UI uses and it might take a short while + ``` npm install ``` ### Compiles and hot-reloads for development +Run this every developing session, this command auto-build the UI after every change your make ``` npm run serve ``` diff --git a/chameleon-client/package-lock.json b/chameleon-client/package-lock.json index 7fbbc6120..6aec554b8 100644 --- a/chameleon-client/package-lock.json +++ b/chameleon-client/package-lock.json @@ -7952,6 +7952,12 @@ "integrity": "sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw==", "dev": true }, + "papaparse": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.1.0.tgz", + "integrity": "sha512-3jEYMiCc8qN7V5ffi2BTS2mRauKxCu5AIED6DxbjnHhIm7OY7fzKYkndfPlHWaaKUDCTml5XTU6V+hiuxGlZuw==", + "dev": true + }, "parallel-transform": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz", diff --git a/chameleon-client/package.json b/chameleon-client/package.json index 44588b6e3..12ab893bc 100644 --- a/chameleon-client/package.json +++ b/chameleon-client/package.json @@ -27,6 +27,7 @@ "babel-eslint": "^10.0.1", "eslint": "^5.16.0", "eslint-plugin-vue": "^5.0.0", + "papaparse": "^5.1.0", "sass": "^1.17.4", "sass-loader": "^7.1.0", "vue-cli-plugin-vuetify": "^0.6.3", diff --git a/chameleon-client/src/assets/chessboard.png b/chameleon-client/src/assets/chessboard.png new file mode 100644 index 000000000..39bb399e8 Binary files /dev/null and b/chameleon-client/src/assets/chessboard.png differ diff --git a/chameleon-client/src/assets/robotIcon.svg b/chameleon-client/src/assets/robotIcon.svg new file mode 100644 index 000000000..8c2f2a247 --- /dev/null +++ b/chameleon-client/src/assets/robotIcon.svg @@ -0,0 +1,140 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/chameleon-client/src/components/3D/MiniMap.vue b/chameleon-client/src/components/3D/MiniMap.vue new file mode 100644 index 000000000..5de4b9b2e --- /dev/null +++ b/chameleon-client/src/components/3D/MiniMap.vue @@ -0,0 +1,165 @@ + + + + + \ No newline at end of file diff --git a/chameleon-client/src/components/OutputTab/DualCalibration.vue b/chameleon-client/src/components/OutputTab/DualCalibration.vue index 22b95ccc5..4513a1cbc 100644 --- a/chameleon-client/src/components/OutputTab/DualCalibration.vue +++ b/chameleon-client/src/components/OutputTab/DualCalibration.vue @@ -40,13 +40,13 @@ if (isNaN(m) === false && isNaN(b) === false) { this.sendSlope(m, b, true); } else { - this.$emit('snackbar'); + this.$emit('snackbar',"Points are too close"); } this.pointA = undefined; this.pointB = undefined; } }, - sendSlope(m, b, valid) { + sendSlope(m, b) { this.handleInput('dualTargetCalibrationM', m); this.handleInput('dualTargetCalibrationB', b); this.$emit('update'); diff --git a/chameleon-client/src/components/OutputTab/SingleCalibration.vue b/chameleon-client/src/components/OutputTab/SingleCalibration.vue index c77dc8b53..75d6edee2 100644 --- a/chameleon-client/src/components/OutputTab/SingleCalibration.vue +++ b/chameleon-client/src/components/OutputTab/SingleCalibration.vue @@ -17,12 +17,16 @@ props: ['rawPoint'], methods: { clearPoint() { - this.handleInput('point', [0, 0]); + this.handleInput('point', []); this.$emit('update'); }, takePoint() { - this.handleInput('point', this.rawPoint); - this.$emit('update'); + if (this.rawPoint[0] && this.rawPoint[1]) { + this.handleInput('point', this.rawPoint); + this.$emit('update'); + } else { + this.$emit('snackbar',"No target found"); + } } } } diff --git a/chameleon-client/src/components/cv-number-input.vue b/chameleon-client/src/components/cv-number-input.vue index 32aaede9b..b2269fd59 100644 --- a/chameleon-client/src/components/cv-number-input.vue +++ b/chameleon-client/src/components/cv-number-input.vue @@ -6,7 +6,7 @@ + style="width: 70px" :step="step"/> @@ -15,7 +15,7 @@ + + \ No newline at end of file diff --git a/chameleon-client/src/views/CameraViewes/InputTab.vue b/chameleon-client/src/views/CameraViewes/InputTab.vue index e45835e0a..f844eb445 100644 --- a/chameleon-client/src/views/CameraViewes/InputTab.vue +++ b/chameleon-client/src/views/CameraViewes/InputTab.vue @@ -46,11 +46,10 @@ streamResolutionList: { get() { let cam_res = this.$store.state.resolutionList[this.value.videoModeIndex]; - let tmp_list = []; - let x = 1; - for (let i = 0; i < 4; i++) { - tmp_list.push(`${cam_res['width'] / x} X ${cam_res['height'] / x}`); - x *= 2; + let tmp_list = []; + tmp_list.push(`${Math.floor(cam_res['width'])} X ${Math.floor(cam_res['height'])}`); + for (let x = 2; x <= 6; x+=2) { + tmp_list.push(`${Math.floor(cam_res['width'] / x)} X ${Math.floor(cam_res['height'] / x)}`); } return tmp_list; } diff --git a/chameleon-client/src/views/CameraViewes/OutputTab.vue b/chameleon-client/src/views/CameraViewes/OutputTab.vue index f9c7f3910..2fe73a691 100644 --- a/chameleon-client/src/views/CameraViewes/OutputTab.vue +++ b/chameleon-client/src/views/CameraViewes/OutputTab.vue @@ -3,14 +3,14 @@ - + Calibrate: - + - Points are too close + {{snackbarText}} Close @@ -40,12 +40,17 @@ }, doUpdate() { this.$emit('update') - } + }, + showSnackbar(message){ + this.snackbarText = message; + this.snackbar = true; + }, }, data() { return { snackbar: false, + snackbarText:"" } }, computed: { diff --git a/chameleon-client/src/views/CameraViewes/ThresholdTab.vue b/chameleon-client/src/views/CameraViewes/ThresholdTab.vue index 4067bb804..979236692 100644 --- a/chameleon-client/src/views/CameraViewes/ThresholdTab.vue +++ b/chameleon-client/src/views/CameraViewes/ThresholdTab.vue @@ -4,36 +4,115 @@ + + + colorize + Eye drop + + + add + Expand Selection + + + remove + Shrink Selection + + \ No newline at end of file diff --git a/chameleon-client/src/views/SettingsViewes/Cameras.vue b/chameleon-client/src/views/SettingsViewes/Cameras.vue index 04d276eaa..ec8ae53ae 100644 --- a/chameleon-client/src/views/SettingsViewes/Cameras.vue +++ b/chameleon-client/src/views/SettingsViewes/Cameras.vue @@ -1,8 +1,57 @@ @@ -17,7 +66,22 @@ CVnumberinput }, data() { - return {} + return { + isCalibrating: false, + resolutionIndex: undefined, + calibrationModeButton: { + text: "Start Calibration", + color: "green" + }, + cancellationModeButton: { + text: "Cancel Calibration", + color: "red" + }, + squareSize: 1.0, + snapshotAmount: 0, + hasEnough: false, + snack: false + } }, methods: { sendCameraSettings() { @@ -30,9 +94,69 @@ } ) }, - + sendCalibrationMode() { + const self = this; + let data = {}; + let connection_string = "/api/settings/"; + if (self.isCalibrating === true) { + connection_string += "snapshot" + } else { + connection_string += "startCalibration"; + data['resolution'] = this.filteredResolutionList[this.resolutionIndex].actualIndex; + data['squareSize'] = this.squareSize; + self.hasEnough = false; + } + this.axios.post("http://" + this.$address + connection_string, data).then( + function (response) { + if (response.status === 200) { + if (self.isCalibrating) { + self.snapshotAmount = response.data['snapshotCount']; + self.hasEnough = response.data['hasEnough']; + if (self.hasEnough === true) { + self.cancellationModeButton.text = "Finish Calibration"; + self.cancellationModeButton.color = "green"; + } + } else { + self.calibrationModeButton.text = "Take Snapshot"; + self.isCalibrating = true; + } + } + } + ); + }, + sendCalibrationFinish() { + const self = this; + let connection_string = "/api/settings/endCalibration"; + let data = {}; + data['squareSize'] = this.squareSize; + self.axios.post("http://" + this.$address + connection_string, data).then( + function (response) { + if (response.status === 500) { + self.snack = true; + } + self.isCalibrating = false; + self.hasEnough = false; + self.snapshotAmount = 0; + self.calibrationModeButton.text = "Start Calibration"; + self.cancellationModeButton.text = "Cancel Calibration"; + self.cancellationModeButton.color = "red"; + } + ); + } }, computed: { + checkResolution() { + return this.resolutionIndex === undefined; + }, + checkCancelation() { + if (this.isCalibrating) { + return false + } else if (this.checkResolution) { + return true; + } else { + return true + } + }, currentCameraIndex: { get() { return this.$store.state.currentCameraIndex; @@ -49,6 +173,28 @@ this.$store.commit('cameraList', value); } }, + filteredResolutionList: { + get() { + let tmp_list = []; + for (let i in this.$store.state.resolutionList) { + let res = JSON.parse(JSON.stringify(this.$store.state.resolutionList[i])); + if (!tmp_list.some(e => e.width === res.width && e.height === res.height)) { + res['actualIndex'] = parseInt(i); + tmp_list.push(res); + } + } + return tmp_list; + } + }, + stringResolutionList: { + get() { + let tmp = []; + for (let i of this.filteredResolutionList) { + tmp.push(`${i['width']} X ${i['height']}`) + } + return tmp + } + }, cameraSettings: { get() { return this.$store.state.cameraSettings; diff --git a/chameleon-server/chameleon-server.iml b/chameleon-server/chameleon-server.iml deleted file mode 100644 index f0b619951..000000000 --- a/chameleon-server/chameleon-server.iml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/chameleon-server/chameleon-vision.iml b/chameleon-server/chameleon-vision.iml index a3c362a46..78fe946ea 100644 --- a/chameleon-server/chameleon-vision.iml +++ b/chameleon-server/chameleon-vision.iml @@ -11,6 +11,20 @@ + + + + + + + + + + + + + + @@ -48,20 +62,20 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/chameleon-server/pom.xml b/chameleon-server/pom.xml index bc1068049..a1deba26e 100644 --- a/chameleon-server/pom.xml +++ b/chameleon-server/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.chameleon-vision.main chameleon-vision - 2.0-RELEASE + 2.1-RELEASE @@ -31,7 +31,8 @@ - + com.chameleonvision.Main @@ -43,18 +44,14 @@ UTF-8 + 2020.1.2 + 3.4.7-2 - - - - - - WPI WPILib Artifactory Server-releases - https://frcmaven.wpi.edu:443/artifactory/development + https://frcmaven.wpi.edu:443/artifactory/release @@ -136,37 +133,37 @@ edu.wpi.first.cscore cscore-java - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} edu.wpi.first.cscore cscore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} linuxaarch64bionic edu.wpi.first.cscore cscore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} linuxraspbian edu.wpi.first.cscore cscore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} linuxx86-64 edu.wpi.first.cscore cscore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} osxx86-64 edu.wpi.first.cscore cscore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} windowsx86-64 @@ -174,45 +171,45 @@ edu.wpi.first.cameraserver cameraserver-java - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} edu.wpi.first.ntcore ntcore-java - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} edu.wpi.first.ntcore ntcore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} osxx86-64 edu.wpi.first.ntcore ntcore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} linuxraspbian edu.wpi.first.ntcore ntcore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} linuxx86-64 edu.wpi.first.ntcore ntcore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} linuxaarch64bionic edu.wpi.first.ntcore ntcore-jni - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} windowsx86-64 @@ -220,43 +217,43 @@ edu.wpi.first.wpiutil wpiutil-java - 2020.1.1-beta-3-12-gb8c1024 + ${wpilib.version} edu.wpi.first.thirdparty.frc2020.opencv opencv-java - 3.4.7-2 + ${opencv.version} edu.wpi.first.thirdparty.frc2020.opencv opencv-jni - 3.4.7-2 + ${opencv.version} linuxaarch64bionic edu.wpi.first.thirdparty.frc2020.opencv opencv-jni - 3.4.7-2 + ${opencv.version} linuxraspbian edu.wpi.first.thirdparty.frc2020.opencv opencv-jni - 3.4.7-2 + ${opencv.version} linuxx86-64 edu.wpi.first.thirdparty.frc2020.opencv opencv-jni - 3.4.7-2 + ${opencv.version} osxx86-64 edu.wpi.first.thirdparty.frc2020.opencv opencv-jni - 3.4.7-2 + ${opencv.version} windowsx86-64 diff --git a/chameleon-server/src/main/java/com/chameleonvision/Exceptions/DuplicatedKeyException.java b/chameleon-server/src/main/java/com/chameleonvision/Exceptions/DuplicatedKeyException.java new file mode 100644 index 000000000..dc54ae95a --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/Exceptions/DuplicatedKeyException.java @@ -0,0 +1,7 @@ +package com.chameleonvision.Exceptions; + +public class DuplicatedKeyException extends Exception{ + public DuplicatedKeyException(String message){ + super(message); + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/Main.java b/chameleon-server/src/main/java/com/chameleonvision/Main.java index 7223da8df..53c8953fa 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/Main.java +++ b/chameleon-server/src/main/java/com/chameleonvision/Main.java @@ -2,7 +2,10 @@ package com.chameleonvision; import com.chameleonvision.config.ConfigManager; import com.chameleonvision.network.NetworkManager; +import com.chameleonvision.scripting.ScriptEventType; +import com.chameleonvision.scripting.ScriptManager; import com.chameleonvision.util.Platform; +import com.chameleonvision.util.ShellExec; import com.chameleonvision.util.Utilities; import com.chameleonvision.vision.VisionManager; import com.chameleonvision.web.Server; @@ -41,6 +44,8 @@ public class Main { if (!hasReportedConnectionFailure && logMessage.message.contains("timed out")) { System.err.println("NT Connection has failed!"); hasReportedConnectionFailure = true; + } else if (logMessage.message.contains("connected")) { + ScriptManager.queueEvent(ScriptEventType.kNTConnected); } } } @@ -101,6 +106,9 @@ public class Main { } public static void main(String[] args) { + + Runtime.getRuntime().addShutdownHook(new Thread(() -> ScriptManager.queueEvent(ScriptEventType.kProgramExit))); + if (CurrentPlatform.equals(Platform.UNSUPPORTED)) { System.err.printf("Sorry, this platform is not supported. Give these details to the developers.\n%s\n", CurrentPlatform.toString()); return; @@ -132,6 +140,14 @@ public class Main { } ConfigManager.initializeSettings(); + + if (!CurrentPlatform.isWindows()) { + ScriptManager.initialize(); + } else { + System.out.println("Scripts not yet supported on Windows. ScriptEvents will be ignored."); + } + + NetworkManager.initialize(manageNetwork); if (ntServerMode) { @@ -147,6 +163,8 @@ public class Main { // NetworkTableInstance.getDefault().startClient("localhost"); } + ScriptManager.queueEvent(ScriptEventType.kProgramInit); + boolean visionSourcesOk = VisionManager.initializeSources(); if (!visionSourcesOk) { System.out.println("No cameras connected!"); @@ -155,13 +173,13 @@ public class Main { boolean visionProcessesOk = VisionManager.initializeProcesses(); if (!visionProcessesOk) { - System.err.println("shit"); + System.err.println("Failed to start threads!"); return; } VisionManager.startProcesses(); - System.out.printf("Starting Webserver at port %d\n", DEFAULT_PORT); + System.out.printf("Starting Web server at port %d\n", DEFAULT_PORT); Server.main(DEFAULT_PORT); } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/config/CameraCalibrationConfig.java b/chameleon-server/src/main/java/com/chameleonvision/config/CameraCalibrationConfig.java new file mode 100644 index 000000000..2dbed87b2 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/config/CameraCalibrationConfig.java @@ -0,0 +1,64 @@ +package com.chameleonvision.config; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreType; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.opencv.core.Mat; +import org.opencv.core.MatOfDouble; +import org.opencv.core.Size; + +/** + * A class that holds a camera matrix and distortion coefficients for a given resolution + */ +public class CameraCalibrationConfig { + @JsonProperty("resolution") public final Size resolution; + @JsonProperty("cameraMatrix") public final JsonMat cameraMatrix; + @JsonProperty("distortionCoeffs") public final JsonMat distortionCoeffs; + @JsonProperty("squareSize") public final double squareSize; + + @JsonCreator + public CameraCalibrationConfig( + @JsonProperty("resolution") Size resolution, + @JsonProperty("cameraMatrix") JsonMat cameraMatrix, + @JsonProperty("distortionCoeffs") JsonMat distortionCoeffs, + @JsonProperty("squareSize") double squareSize) { + this.resolution = resolution; + this.cameraMatrix = cameraMatrix; + this.distortionCoeffs = distortionCoeffs; + this.squareSize = squareSize; + } + + public CameraCalibrationConfig(Size resolution, Mat cameraMatrix, Mat distortionCoeffs, double squareSize) { + this.resolution = resolution; + this.cameraMatrix = JsonMat.fromMat(cameraMatrix); + this.distortionCoeffs = JsonMat.fromMat(distortionCoeffs); + this.squareSize = squareSize; + } + + @JsonIgnoreType + public static class UICameraCalibrationConfig { + public final int width; + public final int height; + public final double[] cameraMatrix; + public final double[] distortionCoeffs; + + public UICameraCalibrationConfig(CameraCalibrationConfig config) { + width = (int) config.resolution.width; + height = (int) config.resolution.height; + cameraMatrix = config.cameraMatrix.data; + distortionCoeffs = config.distortionCoeffs.data; + } + + } + + @JsonIgnore + public Mat getCameraMatrixAsMat() { + return cameraMatrix.toMat(); + } + + @JsonIgnore + public MatOfDouble getDistortionCoeffsAsMat() { + return new MatOfDouble(distortionCoeffs.toMat()); + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/config/CameraConfig.java b/chameleon-server/src/main/java/com/chameleonvision/config/CameraConfig.java index fd0262cdf..66d9df0da 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/config/CameraConfig.java +++ b/chameleon-server/src/main/java/com/chameleonvision/config/CameraConfig.java @@ -1,5 +1,6 @@ package com.chameleonvision.config; +import com.chameleonvision.util.FileHelper; import com.chameleonvision.util.JacksonHelper; import com.chameleonvision.vision.pipeline.CVPipelineSettings; @@ -8,58 +9,85 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class CameraConfig { - private static final Path camerasConfigFolderPath = Paths.get(ConfigManager.SettingsPath.toString(), "cameras"); + private static final Path camerasConfigFolderPath = Path.of(ConfigManager.SettingsPath.toString(), "cameras"); - private final String cameraConfigName; private final CameraJsonConfig preliminaryConfig; + private final Path configFolderPath; + private final Path configPath; + private final Path driverModePath; + private final Path calibrationPath; + final Path pipelineFolderPath; public final PipelineConfig pipelineConfig; CameraConfig(CameraJsonConfig config) { preliminaryConfig = config; - cameraConfigName = preliminaryConfig.name.replace(' ', '_'); + String cameraConfigName = preliminaryConfig.name.replace(' ', '_'); pipelineConfig = new PipelineConfig(this); + + configFolderPath = Path.of(camerasConfigFolderPath.toString(), cameraConfigName); + configPath = Path.of(configFolderPath.toString(), "camera.json"); + driverModePath = Path.of(configFolderPath.toString(), "drivermode.json"); + calibrationPath = Path.of(configFolderPath.toString(), "calibration.json"); + pipelineFolderPath = Paths.get(configFolderPath.toString(), "pipelines"); } public FullCameraConfiguration load() { checkFolder(); checkConfig(); checkDriverMode(); + checkCalibration(); pipelineConfig.check(); - return new FullCameraConfiguration(loadConfig(), pipelineConfig.load(), loadDriverMode(), this); + return new FullCameraConfiguration(loadConfig(), pipelineConfig.load(), loadDriverMode(), loadCalibration(), this); } private CameraJsonConfig loadConfig() { CameraJsonConfig config = preliminaryConfig; try { - config = JacksonHelper.deserializer(getConfigPath(), CameraJsonConfig.class); + config = JacksonHelper.deserializer(configPath, CameraJsonConfig.class); } catch (IOException e) { - System.err.printf("Failed to load camera config: %s - using default.\n", getConfigPath().toString()); + System.err.printf("Failed to load camera config: %s - using default.\n", configPath.toString()); } return config; } private CVPipelineSettings loadDriverMode() { CVPipelineSettings driverMode = new CVPipelineSettings(); - driverMode.nickname = "DRIVERMODE"; try { - driverMode = JacksonHelper.deserializer(getDriverModePath(), CVPipelineSettings.class); + driverMode = JacksonHelper.deserializer(driverModePath, CVPipelineSettings.class); } catch (IOException e) { - System.err.println("Failed to load camera drivermode: " + getDriverModePath().toString()); + System.err.println("Failed to load camera drivermode: " + driverModePath.toString()); + } + if (driverMode != null) { + driverMode.nickname = "DRIVERMODE"; + driverMode.index = -1; } return driverMode; } + private List loadCalibration() { + List calibrations = new ArrayList<>(); + try { + calibrations = List.of(Objects.requireNonNull(JacksonHelper.deserializer(calibrationPath, CameraCalibrationConfig[].class))); + } catch (Exception e) { + System.err.println("Failed to load camera calibration: " + driverModePath.toString()); + } + return calibrations; + } + void saveConfig(CameraJsonConfig config) { try { - JacksonHelper.serializer(getConfigPath(), config); + JacksonHelper.serializer(configPath, config); + FileHelper.setFilePerms(configPath); } catch (IOException e) { - System.err.println("Failed to save camera config file: " + getConfigPath().toString()); + System.err.println("Failed to save camera config file: " + configPath.toString()); } } @@ -69,20 +97,33 @@ public class CameraConfig { public void saveDriverMode(CVPipelineSettings driverMode) { try { - JacksonHelper.serializer(getDriverModePath(), driverMode); + JacksonHelper.serializer(driverModePath, driverMode); + FileHelper.setFilePerms(driverModePath); } catch (IOException e) { - System.err.println("Failed to save camera drivermode file: " + getDriverModePath().toString()); + System.err.println("Failed to save camera drivermode file: " + driverModePath.toString()); + } + } + + + public void saveCalibration(List cal) { + CameraCalibrationConfig[] configs = cal.toArray(new CameraCalibrationConfig[0]); + try { + JacksonHelper.serializer(calibrationPath, configs); + FileHelper.setFilePerms(calibrationPath); + } catch (IOException e) { + System.err.println("Failed to save camera calibration file: " + calibrationPath.toString()); } } void checkFolder() { - if (!getConfigFolderExists()) { + if (!configFolderExists()) { try { - if (!(new File(getConfigFolderPath().toUri()).mkdirs())) { - System.err.println("Failed to create camera config folder: " + getConfigFolderPath().toString()); + if (!(new File(configFolderPath.toUri()).mkdirs())) { + System.err.println("Failed to create camera config folder: " + configFolderPath.toString()); } + FileHelper.setFilePerms(configFolderPath); } catch(Exception e) { - System.err.println("Failed to create camera config folder: " + getConfigFolderPath().toString()); + System.err.println("Failed to create camera config folder: " + configFolderPath.toString()); } } } @@ -90,9 +131,10 @@ public class CameraConfig { private void checkConfig() { if (!configExists()) { try { - JacksonHelper.serializer(getConfigPath(), preliminaryConfig); + JacksonHelper.serializer(configPath, preliminaryConfig); + FileHelper.setFilePerms(configPath); } catch (IOException e) { - System.err.println("Failed to create camera config file: " + getConfigPath().toString()); + System.err.println("Failed to create camera config file: " + configPath.toString()); } } } @@ -102,38 +144,38 @@ public class CameraConfig { try { CVPipelineSettings newDriverModeSettings = new CVPipelineSettings(); newDriverModeSettings.nickname = "DRIVERMODE"; - JacksonHelper.serializer(getDriverModePath(), newDriverModeSettings); + JacksonHelper.serializer(driverModePath, newDriverModeSettings); + FileHelper.setFilePerms(driverModePath); } catch (IOException e) { - System.err.println("Failed to create camera drivermode file: " + getDriverModePath().toString()); + System.err.println("Failed to create camera drivermode file: " + driverModePath.toString()); } } } - private Path getConfigFolderPath() { - return Paths.get(camerasConfigFolderPath.toString(), cameraConfigName); + private void checkCalibration() { + if (!calibrationExists()) { + try { + List calibrations = new ArrayList<>(); + JacksonHelper.serializer(calibrationPath, calibrations.toArray()); + } catch (IOException e) { + System.err.println("Failed to create camera calibration file: " + calibrationPath.toString()); + } + } } - private Path getConfigPath() { - return Paths.get(getConfigFolderPath().toString(), "camera.json"); - } - - private Path getDriverModePath() { - return Paths.get(getConfigFolderPath().toString(), "drivermode.json"); - } - - private boolean getConfigFolderExists() { - return Files.exists(getConfigFolderPath()); - } - - Path getPipelineFolderPath() { - return Paths.get(getConfigFolderPath().toString(), "pipelines"); + private boolean configFolderExists() { + return Files.exists(configFolderPath); } private boolean configExists() { - return getConfigFolderExists() && Files.exists(getConfigPath()); + return configFolderExists() && Files.exists(configPath); } private boolean driverModeExists() { - return getConfigFolderExists() && Files.exists(getDriverModePath()); + return configFolderExists() && Files.exists(driverModePath); + } + + private boolean calibrationExists() { + return configFolderExists() && Files.exists(calibrationPath); } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/config/ConfigManager.java b/chameleon-server/src/main/java/com/chameleonvision/config/ConfigManager.java index 3003b6d02..18da8285e 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/config/ConfigManager.java +++ b/chameleon-server/src/main/java/com/chameleonvision/config/ConfigManager.java @@ -1,7 +1,7 @@ package com.chameleonvision.config; -import com.chameleonvision.util.ProgramDirectoryUtilities; -import com.chameleonvision.util.JacksonHelper; +import com.chameleonvision.Main; +import com.chameleonvision.util.*; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import java.io.File; @@ -16,7 +16,7 @@ import java.util.List; public class ConfigManager { private ConfigManager() {} - static final Path SettingsPath = Paths.get(ProgramDirectoryUtilities.getProgramDirectory(), "settings"); + public static final Path SettingsPath = Paths.get(ProgramDirectoryUtilities.getProgramDirectory(), "settings"); private static final Path settingsFilePath = Paths.get(SettingsPath.toString(), "settings.json"); private static final LinkedHashMap cameraConfigs = new LinkedHashMap<>(); @@ -33,6 +33,9 @@ public class ConfigManager { System.err.println("Failed to create settings folder: " + SettingsPath.toString()); } Files.createDirectory(SettingsPath); + if (!Platform.CurrentPlatform.isWindows()) { + new ShellExec().executeBashCommand("sudo chmod -R 0777 " + SettingsPath.toString()); + } } catch (IOException e) { if(!(e instanceof java.nio.file.FileAlreadyExistsException)) e.printStackTrace(); @@ -45,6 +48,7 @@ public class ConfigManager { if (settingsFileEmpty || !settingsFileExists()) { try { JacksonHelper.serializer(settingsFilePath, settings); + FileHelper.setFilePerms(settingsFilePath); } catch (IOException e) { e.printStackTrace(); } @@ -61,11 +65,13 @@ public class ConfigManager { System.out.println("Settings folder: " + SettingsPath.toString()); checkSettingsFolder(); checkSettingsFile(); + FileHelper.setAllPerms(SettingsPath); } private static void saveSettingsFile() { try { JacksonHelper.serializer(settingsFilePath, settings); + FileHelper.setFilePerms(settingsFilePath); } catch (IOException e) { System.err.println("Failed to save settings.json!"); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java b/chameleon-server/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java index 7e98c09b4..632526fbc 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java +++ b/chameleon-server/src/main/java/com/chameleonvision/config/FullCameraConfiguration.java @@ -7,13 +7,15 @@ import java.util.List; public class FullCameraConfiguration { public final CameraJsonConfig cameraConfig; public final List pipelines; - public final CVPipelineSettings drivermode; + public final CVPipelineSettings driverMode; + public final List calibration; public final CameraConfig fileConfig; - FullCameraConfiguration(CameraJsonConfig cameraConfig, List pipelines, CVPipelineSettings drivermode, CameraConfig fileConfig) { + FullCameraConfiguration(CameraJsonConfig cameraConfig, List pipelines, CVPipelineSettings driverMode, List calibration, CameraConfig fileConfig) { this.cameraConfig = cameraConfig; this.pipelines = pipelines; - this.drivermode = drivermode; + this.driverMode = driverMode; + this.calibration = calibration; this.fileConfig = fileConfig; } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/config/JsonMat.java b/chameleon-server/src/main/java/com/chameleonvision/config/JsonMat.java new file mode 100644 index 000000000..df1fded86 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/config/JsonMat.java @@ -0,0 +1,78 @@ +package com.chameleonvision.config; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.opencv.core.CvType; +import org.opencv.core.Mat; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + + +public class JsonMat { + public final int rows; + public final int cols; + public final int type; + public final double[] data; + + public JsonMat(int rows, int cols, double[] data) { + this(rows, cols, CvType.CV_64FC1, data); + } + + public JsonMat( + @JsonProperty("rows") int rows, + @JsonProperty("cols") int cols, + @JsonProperty("type") int type, + @JsonProperty("data") double[] data) { + this.rows = rows; + this.cols = cols; + this.type = type; + this.data = data; + } + + public Mat toMat() { + return toMat(this); + } + + private static boolean isCameraMatrixMat(Mat mat) { + return mat.type() == CvType.CV_64FC1 && mat.cols() == 3 && mat.rows() == 3; + } + + private static boolean isDistortionCoeffsMat(Mat mat) { + return mat.type() == CvType.CV_64FC1 && mat.cols() == 5 && mat.rows() == 1; + } + + private static boolean isCalibrationMat(Mat mat) { + return isDistortionCoeffsMat(mat) || isCameraMatrixMat(mat); + } + + public static double[] getDataFromMat(Mat mat) { + if (!isCalibrationMat(mat)) return null; + + double[] data = new double[(int)(mat.total()*mat.elemSize())]; + mat.get(0, 0, data); + + int dataLen = -1; + + if (isCameraMatrixMat(mat)) dataLen = 9; + if (isDistortionCoeffsMat(mat)) dataLen = 5; + + // truncate Mat data to correct number data points. + return Arrays.copyOfRange(data, 0, dataLen); + } + + public static JsonMat fromMat(Mat mat) { + if (!isCalibrationMat(mat)) return null; + return new JsonMat(mat.rows(), mat.cols(), getDataFromMat(mat)); + } + + public static Mat toMat(JsonMat jsonMat) { + if (jsonMat.type != CvType.CV_64FC1) return null; + + Mat retMat = new Mat(jsonMat.rows, jsonMat.cols, jsonMat.type); + retMat.put(0, 0, jsonMat.data); + return retMat; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/config/PipelineConfig.java b/chameleon-server/src/main/java/com/chameleonvision/config/PipelineConfig.java index e57ca250f..e4d1c4757 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/config/PipelineConfig.java +++ b/chameleon-server/src/main/java/com/chameleonvision/config/PipelineConfig.java @@ -1,7 +1,9 @@ package com.chameleonvision.config; +import com.chameleonvision.util.FileHelper; import com.chameleonvision.util.JacksonHelper; import com.chameleonvision.vision.pipeline.*; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipelineSettings; import java.io.File; import java.io.IOException; @@ -13,9 +15,6 @@ import java.util.List; public class PipelineConfig { - private static final String CVPipeline2DPrefix = "CV2D"; - private static final String CVPipeline3DPrefix = "CV3D"; - private final CameraConfig cameraConfig; /** @@ -27,15 +26,20 @@ public class PipelineConfig { } private void checkFolder() { - if ( !(new File(cameraConfig.getPipelineFolderPath().toUri()).mkdirs())) { - if (Files.notExists(cameraConfig.getPipelineFolderPath())) { + if ( !(new File(cameraConfig.pipelineFolderPath.toUri()).mkdirs())) { + if (Files.notExists(cameraConfig.pipelineFolderPath)) { System.err.println("Failed to create pipelines folder."); } } + try { + FileHelper.setFilePerms(cameraConfig.pipelineFolderPath); + } catch (IOException e) { + // ignored + } } private File[] getPipelineFiles() { - return new File(cameraConfig.getPipelineFolderPath().toUri()).listFiles(); + return new File(cameraConfig.pipelineFolderPath.toUri()).listFiles(); } private boolean folderHasPipelines() { @@ -49,15 +53,14 @@ public class PipelineConfig { checkFolder(); // Check if there's at least one pipe if (!folderHasPipelines()) { - save(new CVPipeline2dSettings()); + save(new StandardCVPipelineSettings()); } } private Path getPipelinePath(CVPipelineSettings setting) { String pipelineName = setting.nickname.replace(' ', '_'); - String prefix = ((setting instanceof CVPipeline2dSettings) ? CVPipeline2DPrefix : CVPipeline3DPrefix) + "-"; - String fullFileName = prefix + pipelineName + ".json"; - return Path.of(cameraConfig.getPipelineFolderPath().toString(), fullFileName); + String fullFileName = pipelineName + ".json"; + return Path.of(cameraConfig.pipelineFolderPath.toString(), fullFileName); } private boolean pipelineExists(CVPipelineSettings setting) { @@ -68,20 +71,11 @@ public class PipelineConfig { var path = getPipelinePath(settings); - if (settings instanceof CVPipeline3dSettings) { - try { - JacksonHelper.serializer(path, settings); - } catch (IOException e) { - e.printStackTrace(); - } - } else if (settings instanceof CVPipeline2dSettings) { - try { - JacksonHelper.serializer(path, settings); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - throw new RuntimeException("saving non-2d and non-3d pipelines not implemented~"); + try { + JacksonHelper.serializer(path, settings); + FileHelper.setFilePerms(path); + } catch (IOException e) { + e.printStackTrace(); } } @@ -124,24 +118,12 @@ public class PipelineConfig { System.err.println("no pipes to load! loading default"); } else { for(File pipelineFile : pipelineFiles) { - var name = pipelineFile.getName(); - if(name.startsWith(CVPipeline3DPrefix)) { - // try to load 3d pipe try { - var pipe = JacksonHelper.deserializer(Paths.get(pipelineFile.getPath()), CVPipeline3dSettings.class); - deserializedList.add(pipe); - } catch (IOException e) { - System.err.println("couldn't load cvpipeline3d"); - } - } else if(name.startsWith(CVPipeline2DPrefix)) { - // try to load 2d pipe - try { - var pipe = JacksonHelper.deserializer(Paths.get(pipelineFile.getPath()), CVPipeline2dSettings.class); + var pipe = JacksonHelper.deserializer(Paths.get(pipelineFile.getPath()), StandardCVPipelineSettings.class); deserializedList.add(pipe); } catch (IOException e) { System.err.println("couldn't load cvpipeline2d"); } - } } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/network/NetworkManager.java b/chameleon-server/src/main/java/com/chameleonvision/network/NetworkManager.java index f9cd51eb4..d505199cd 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/network/NetworkManager.java +++ b/chameleon-server/src/main/java/com/chameleonvision/network/NetworkManager.java @@ -20,7 +20,7 @@ public class NetworkManager { return; } - Platform platform = Platform.getCurrentPlatform(); + Platform platform = Platform.CurrentPlatform; if (platform.isLinux()) { networking = new LinuxNetworking(); diff --git a/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptCommandType.java b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptCommandType.java new file mode 100644 index 000000000..017512065 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptCommandType.java @@ -0,0 +1,14 @@ +package com.chameleonvision.scripting; + +public enum ScriptCommandType { + kDefault(""), + kBashScript("bash"), + kPythonScript("python"), + kPython3Script("python3"); + + public final String value; + + ScriptCommandType(String value) { + this.value = value; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptConfig.java b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptConfig.java new file mode 100644 index 000000000..313397841 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptConfig.java @@ -0,0 +1,23 @@ +package com.chameleonvision.scripting; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ScriptConfig { + public final ScriptEventType eventType; + public final String command; + + public ScriptConfig(ScriptEventType eventType) { + this.eventType = eventType; + this.command = ""; + } + + @JsonCreator + public ScriptConfig( + @JsonProperty("eventType") ScriptEventType eventType, + @JsonProperty("command") String command + ) { + this.eventType = eventType; + this.command = command; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptEvent.java b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptEvent.java new file mode 100644 index 000000000..c117c73de --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptEvent.java @@ -0,0 +1,35 @@ +package com.chameleonvision.scripting; + +import com.chameleonvision.Debug; +import com.chameleonvision.util.ShellExec; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +public class ScriptEvent { + private static final ShellExec executor = new ShellExec(true, true); + + public final ScriptConfig config; + + public ScriptEvent(ScriptConfig config) { + this.config = config; + } + + public int run() throws IOException { + int retVal = executor.executeBashCommand(config.command); + + String output = executor.getOutput(); + String error = executor.getError(); + + if (!error.isEmpty()) { + System.err.printf("Error when running \"%s\" script: %s\n", config.eventType.name(), error); + } else if (!output.isEmpty()) { + Debug.printInfo(String.format("Output from \"%s\" script: %s\n", config.eventType.name(), output)); + } + Debug.printInfo(String.format("Script for %s ran with command line: \"%s\", exit code: %d, output: %s, error: %s\n", config.eventType.name(), config.command, retVal, output, error)); + return retVal; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptEventType.java b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptEventType.java new file mode 100644 index 000000000..436676ee3 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptEventType.java @@ -0,0 +1,21 @@ +package com.chameleonvision.scripting; + +public enum ScriptEventType { + kProgramInit("Program Init"), + kProgramExit("Program Exit"), + kNTConnected("NT Connected"), + kLEDOn("LED On"), + kLEDOff("LED Off"), + kEnterDriverMode("Enter Driver Mode"), + kExitDriverMode("Exit Driver Mode"), + kFoundTarget("Found Target"), + kFoundMultipleTarget("Found Multiple Target"), + kLostTarget("Lost Target"), + kPipelineLag("Pipeline Lag"); + + public final String value; + + ScriptEventType(String value) { + this.value = value; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptManager.java b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptManager.java new file mode 100644 index 000000000..9385a13c3 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/scripting/ScriptManager.java @@ -0,0 +1,126 @@ +package com.chameleonvision.scripting; + +import com.chameleonvision.Debug; +import com.chameleonvision.Main; +import com.chameleonvision.config.ConfigManager; +import com.chameleonvision.util.JacksonHelper; +import com.chameleonvision.util.LoopingRunnable; +import com.chameleonvision.util.Platform; +import com.chameleonvision.util.ProgramDirectoryUtilities; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.LinkedBlockingQueue; + +public class ScriptManager { + + private ScriptManager() {} + + private static final List events = new ArrayList<>(); + private static final LinkedBlockingDeque queuedEvents = new LinkedBlockingDeque<>(25); + + public static void initialize() { + ScriptConfigManager.initialize(); + if (ScriptConfigManager.fileExists()) { + for (ScriptConfig scriptConfig : ScriptConfigManager.loadConfig()) { + ScriptEvent scriptEvent = new ScriptEvent(scriptConfig); + events.add(scriptEvent); + } + + new Thread(new ScriptRunner(10L)).start(); + } else { + System.err.println("Something went wrong initializing scripts! Events will not run."); + } + } + + private static class ScriptRunner extends LoopingRunnable { + + ScriptRunner(Long loopTimeMs) { + super(loopTimeMs); + } + + @Override + protected void process() { + try { + + handleEvent(queuedEvents.takeFirst()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + private void handleEvent(ScriptEventType eventType) { + var toRun = events.parallelStream().filter(e -> e.config.eventType == eventType).findFirst().orElse(null); + if (toRun != null) { + try { + toRun.run(); + } catch (IOException e) { + System.err.printf("Failed to run script for event: %s, exception below.\n%s\n", eventType.name(), e.getMessage()); + } + } + } + } + + protected static class ScriptConfigManager { + + protected static final Path scriptConfigPath = Paths.get(ConfigManager.SettingsPath.toString(), "scripts.json"); + + private ScriptConfigManager() {} + + static boolean fileExists() { return Files.exists(scriptConfigPath); } + + public static void initialize() { + if (!fileExists()) { + List eventsConfig = new ArrayList<>(); + for (var eventType : ScriptEventType.values()) { + eventsConfig.add(new ScriptConfig(eventType)); + } + + try { + JacksonHelper.serializer(scriptConfigPath, eventsConfig.toArray(new ScriptConfig[0])); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + static List loadConfig() { + try { + var raw = JacksonHelper.deserializer(scriptConfigPath, ScriptConfig[].class); + if (raw != null) { + return List.of(raw); + } + } catch (IOException e) { + e.printStackTrace(); + } + return new ArrayList<>(); + } + + protected static void deleteConfig() { + try { + Files.delete(scriptConfigPath); + } catch (IOException e) { + // + } + } + } + + public static void queueEvent(ScriptEventType eventType) { + if (!Platform.getCurrentPlatform().isWindows()) { + try { + queuedEvents.putLast(eventType); + Debug.printInfo("Queued event: " + eventType.name()); + } catch (InterruptedException e) { + System.err.println("Failed to add event to queue: " + eventType.name()); + } + } + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/util/FileHelper.java b/chameleon-server/src/main/java/com/chameleonvision/util/FileHelper.java new file mode 100644 index 000000000..1ff3c139c --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/util/FileHelper.java @@ -0,0 +1,44 @@ +package com.chameleonvision.util; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class FileHelper { + private FileHelper() {} + + private static final Set allReadWriteExecutePerms = new HashSet<>(Arrays.asList(PosixFilePermission.values())); + + public static void setFilePerms(Path path) throws IOException { + if (!Platform.CurrentPlatform.isWindows()) { + File thisFile = path.toFile(); + Set perms = Files.readAttributes(path, PosixFileAttributes.class).permissions(); + if (!perms.equals(allReadWriteExecutePerms)) { + System.out.printf("setting perms on %s\n", path.toString()); + Files.setPosixFilePermissions(path, perms); + if (thisFile.isDirectory()) { + for (File subfile : thisFile.listFiles()) { + setFilePerms(subfile.toPath()); + } + } + } + } + } + + public static void setAllPerms(Path path) { + String command = String.format("chmod 777 -R %s", path.toString()); + try { + Process p = Runtime.getRuntime().exec(command); + p.waitFor(); + + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/util/JacksonHelper.java b/chameleon-server/src/main/java/com/chameleonvision/util/JacksonHelper.java index b0a856c69..8979b4621 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/util/JacksonHelper.java +++ b/chameleon-server/src/main/java/com/chameleonvision/util/JacksonHelper.java @@ -4,9 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator; import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator; + import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; public class JacksonHelper { diff --git a/chameleon-server/src/main/java/com/chameleonvision/util/MathHandler.java b/chameleon-server/src/main/java/com/chameleonvision/util/MathHandler.java index a95a089e3..f2bcbaa62 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/util/MathHandler.java +++ b/chameleon-server/src/main/java/com/chameleonvision/util/MathHandler.java @@ -24,4 +24,9 @@ public class MathHandler { public static double toSlope(Number angle){ return FastMath.atan(FastMath.toRadians(angle.doubleValue() - 90)); } + + public static double roundTo(double value, int to) { + double toMult = Math.pow(10, to); + return (double)Math.round(value * toMult) / toMult; + } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/util/Platform.java b/chameleon-server/src/main/java/com/chameleonvision/util/Platform.java index 4bc4e6712..203880e98 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/util/Platform.java +++ b/chameleon-server/src/main/java/com/chameleonvision/util/Platform.java @@ -23,7 +23,7 @@ public enum Platform { private static final String OS_ARCH = System.getProperty("os.arch"); public static final Platform CurrentPlatform = getCurrentPlatform(); - public boolean isWindows() { + public boolean isWindows() { return this == WINDOWS_64; } @@ -35,6 +35,10 @@ public enum Platform { return this == MACOS_64; } + public static boolean isRaspberryPi() { + return CurrentPlatform.equals(LINUX_RASPBIAN); + } + private static ShellExec shell = new ShellExec(true, false); public boolean isRoot() { diff --git a/chameleon-server/src/main/java/com/chameleonvision/util/ProgramDirectoryUtilities.java b/chameleon-server/src/main/java/com/chameleonvision/util/ProgramDirectoryUtilities.java index b78b45e91..3c415a412 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/util/ProgramDirectoryUtilities.java +++ b/chameleon-server/src/main/java/com/chameleonvision/util/ProgramDirectoryUtilities.java @@ -24,19 +24,16 @@ public class ProgramDirectoryUtilities { if (runningFromJAR()) { + if (Platform.isRaspberryPi()) { + return "/boot/chameleon-vision"; + } return getCurrentJARDirectory(); } else { return System.getProperty("user.dir"); -// return getCurrentProjectDirectory(); } } - private static String getCurrentProjectDirectory() - { - return new File("").getAbsolutePath(); - } - private static String getCurrentJARDirectory() { try diff --git a/chameleon-server/src/main/java/com/chameleonvision/util/ShellExec.java b/chameleon-server/src/main/java/com/chameleonvision/util/ShellExec.java index f0fa0f1cd..60856cf79 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/util/ShellExec.java +++ b/chameleon-server/src/main/java/com/chameleonvision/util/ShellExec.java @@ -19,6 +19,43 @@ public class ShellExec { this.readError = readError; } + /** + * Execute a bash command. We can handle complex bash commands including + * multiple executions (; | && ||), quotes, expansions ($), escapes (\), e.g.: + * "cd /abc/def; mv ghi 'older ghi '$(whoami)" + * @param command + * @return true if bash got started, but your command may have failed. + */ + public int executeBashCommand(String command) throws IOException { + boolean wait = true; + boolean success = false; + Runtime r = Runtime.getRuntime(); + // Use bash -c so we can handle things like multi commands separated by ; and + // things like quotes, $, |, and \. My tests show that command comes as + // one argument to bash, so we do not need to quote it to make it one thing. + // Also, exec may object if it does not have an executable file as the first thing, + // so having bash here makes it happy provided bash is installed and in path. + String[] commands = {"bash", "-c", command}; + + Process process = r.exec(commands); + + // Consume streams, older jvm's had a memory leak if streams were not read, + // some other jvm+OS combinations may block unless streams are consumed. + errorGobbler = new StreamGobbler(process.getErrorStream(), readError); + outputGobbler = new StreamGobbler(process.getInputStream(), readOutput); + errorGobbler.start(); + outputGobbler.start(); + + exitCode = 0; + if (wait) { + try { + process.waitFor(); + exitCode = process.exitValue(); + } catch (InterruptedException ignored) { } + } + return exitCode; + } + /** * Execute a command in current folder, and wait for process to end * @param command command ("c:/some/folder/script.bat" or "some/folder/script.sh") diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/VisionManager.java b/chameleon-server/src/main/java/com/chameleonvision/vision/VisionManager.java index d4d42c662..dda752965 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/VisionManager.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/VisionManager.java @@ -6,7 +6,6 @@ import com.chameleonvision.config.ConfigManager; import com.chameleonvision.config.FullCameraConfiguration; import com.chameleonvision.util.Helpers; import com.chameleonvision.util.Platform; -import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.vision.camera.USBCameraCapture; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import edu.wpi.cscore.UsbCamera; @@ -84,9 +83,9 @@ public class VisionManager { CameraJsonConfig cameraJsonConfig = config.cameraConfig; - USBCameraCapture camera = new USBCameraCapture(cameraJsonConfig); - VisionProcess process = new VisionProcess(camera, cameraJsonConfig.name, config.pipelines); - process.pipelineManager.driverModePipeline.settings = config.drivermode; + USBCameraCapture camera = new USBCameraCapture(config); + VisionProcess process = new VisionProcess(camera, config); + process.pipelineManager.driverModePipeline.settings = config.driverMode; visionProcesses.add(new VisionProcessManageable(i, cameraJsonConfig.name, process)); } currentUIVisionProcess = getVisionProcessByIndex(0); @@ -102,6 +101,10 @@ public class VisionManager { return currentUIVisionProcess; } + public static CameraConfig getCurrentCameraConfig() { + return getCameraConfig(currentUIVisionProcess); + } + public static CameraConfig getCameraConfig(VisionProcess process) { String cameraName = process.getCamera().getProperties().name; return Objects.requireNonNull(loadedCameraConfigs.stream().filter(x -> x.cameraConfig.name.equals(cameraName)).findFirst().orElse(null)).fileConfig; diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java b/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java index 9d11db4fd..f501e8ee9 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/VisionProcess.java @@ -1,17 +1,26 @@ package com.chameleonvision.vision; import com.chameleonvision.Debug; +import com.chameleonvision.config.CameraCalibrationConfig; +import com.chameleonvision.config.CameraConfig; import com.chameleonvision.config.ConfigManager; +import com.chameleonvision.scripting.ScriptEventType; +import com.chameleonvision.scripting.ScriptManager; +import com.chameleonvision.config.FullCameraConfiguration; import com.chameleonvision.util.LoopingRunnable; -import com.chameleonvision.vision.camera.CameraCapture; +import com.chameleonvision.util.MathHandler; import com.chameleonvision.vision.camera.CameraStreamer; import com.chameleonvision.vision.camera.USBCameraCapture; import com.chameleonvision.vision.pipeline.*; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; +import com.chameleonvision.vision.pipeline.impl.DriverVisionPipeline; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipelineSettings; import com.chameleonvision.web.SocketHandler; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import edu.wpi.cscore.VideoMode; import edu.wpi.first.networktables.*; +import edu.wpi.first.wpilibj.geometry.Pose2d; import edu.wpi.first.wpiutil.CircularBuffer; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; @@ -21,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; +import java.util.stream.Collectors; public class VisionProcess { @@ -28,6 +38,7 @@ public class VisionProcess { private final USBCameraCapture cameraCapture; private final CameraStreamerRunnable streamRunnable; private final VisionProcessRunnable visionRunnable; + private final CameraConfig fileConfig; public final CameraStreamer cameraStreamer; public PipelineManager pipelineManager; @@ -37,6 +48,7 @@ public class VisionProcess { // network table stuff private final NetworkTable defaultTable; + private NetworkTableInstance tableInstance; private NetworkTableEntry ntPipelineEntry; public NetworkTableEntry ntDriverModeEntry; private int ntDriveModeListenerID; @@ -45,17 +57,22 @@ public class VisionProcess { private NetworkTableEntry ntPitchEntry; private NetworkTableEntry ntAuxListEntry; private NetworkTableEntry ntAreaEntry; - private NetworkTableEntry ntTimeStampEntry; + private NetworkTableEntry ntLatencyEntry; private NetworkTableEntry ntValidEntry; + private NetworkTableEntry ntPoseEntry; private ObjectMapper objectMapper = new ObjectMapper(); - VisionProcess(USBCameraCapture cameraCapture, String name, List loadedPipelineSettings) { + private long lastUIUpdateMs = 0; + + VisionProcess(USBCameraCapture cameraCapture, FullCameraConfiguration config) { this.cameraCapture = cameraCapture; - pipelineManager = new PipelineManager(this, loadedPipelineSettings); + fileConfig = config.fileConfig; + + pipelineManager = new PipelineManager(this, config.pipelines); // Thread to put frames on the dashboard - this.cameraStreamer = new CameraStreamer(cameraCapture, name); + this.cameraStreamer = new CameraStreamer(cameraCapture, config.cameraConfig.name, pipelineManager.getCurrentPipeline().settings.streamDivisor); this.streamRunnable = new CameraStreamerRunnable(30, cameraStreamer); // Thread to process vision data @@ -74,10 +91,10 @@ public class VisionProcess { visionThread.setName(getCamera().getProperties().name + " - Vision Thread"); visionThread.start(); - System.out.println("Starting stream thread."); - var streamThread = new Thread(streamRunnable); - streamThread.setName(getCamera().getProperties().name + " - Stream Thread"); - streamThread.start(); +// System.out.println("Starting stream thread."); +// var streamThread = new Thread(streamRunnable); +// streamThread.setName(getCamera().getProperties().name + " - Stream Thread"); +// streamThread.start(); } /** @@ -99,14 +116,16 @@ public class VisionProcess { } private void initNT(NetworkTable newTable) { + tableInstance = newTable.getInstance(); ntPipelineEntry = newTable.getEntry("pipeline"); ntDriverModeEntry = newTable.getEntry("driver_mode"); ntPitchEntry = newTable.getEntry("pitch"); ntYawEntry = newTable.getEntry("yaw"); ntAreaEntry = newTable.getEntry("area"); - ntTimeStampEntry = newTable.getEntry("timestamp"); + ntLatencyEntry = newTable.getEntry("latency"); ntValidEntry = newTable.getEntry("is_valid"); ntAuxListEntry = newTable.getEntry("aux_targets"); + ntPoseEntry = newTable.getEntry("poseList"); ntDriveModeListenerID = ntDriverModeEntry.addListener(this::setDriverMode, EntryListenerFlags.kUpdate); ntPipelineListenerID = ntPipelineEntry.addListener(this::setPipeline, EntryListenerFlags.kUpdate); ntDriverModeEntry.setBoolean(false); @@ -120,11 +139,13 @@ public class VisionProcess { public void setDriverMode(boolean driverMode) { pipelineManager.setDriverMode(driverMode); + ScriptManager.queueEvent(driverMode ? ScriptEventType.kEnterDriverMode : ScriptEventType.kExitDriverMode); SocketHandler.sendFullSettings(); } /** * Method called by the nt entry listener to update the next pipeline. + * * @param notification the notification */ private void setPipeline(EntryNotification notification) { @@ -136,76 +157,101 @@ public class VisionProcess { // if it's null, we haven't even started the program yet, so just return // otherwise, set it. - if(ntDriverModeEntry != null) { + if (ntDriverModeEntry != null) { ntDriverModeEntry.setBoolean(isDriverMode); } } private void updateUI(CVPipelineResult data) { - if(cameraCapture.getProperties().name.equals(ConfigManager.settings.currentCamera)) { - HashMap WebSend = new HashMap<>(); - HashMap point = new HashMap<>(); - HashMap calculated = new HashMap<>(); - List center = new ArrayList<>(); - if (data.hasTarget) { - if(data instanceof CVPipeline2d.CVPipeline2dResult) { - CVPipeline2d.CVPipeline2dResult result = (CVPipeline2d.CVPipeline2dResult) data; - CVPipeline2d.Target2d bestTarget = result.targets.get(0); - center.add(bestTarget.rawPoint.center.x); - center.add(bestTarget.rawPoint.center.y); - calculated.put("pitch", bestTarget.pitch); - calculated.put("yaw", bestTarget.yaw); - calculated.put("area", bestTarget.area); - } else if (data instanceof CVPipeline3d.CVPipeline3dResult) { - // TODO: (2.1) 3d stuff in UI + // 30 "FPS" update rate + long currentMillis = System.currentTimeMillis(); + if (currentMillis - lastUIUpdateMs > 1000 / 30) { + lastUIUpdateMs = currentMillis; + + + if (cameraCapture.getProperties().name.equals(ConfigManager.settings.currentCamera)) { + HashMap WebSend = new HashMap<>(); + HashMap point = new HashMap<>(); + HashMap pointMap = new HashMap<>(); + ArrayList webTargets = new ArrayList(); + List center = new ArrayList<>(); + + + + if (data.hasTarget) { + if (data instanceof StandardCVPipeline.StandardCVPipelineResult) { + StandardCVPipeline.StandardCVPipelineResult result = (StandardCVPipeline.StandardCVPipelineResult) data; + StandardCVPipeline.TrackedTarget bestTarget = result.targets.get(0); + if (((StandardCVPipelineSettings) pipelineManager.getCurrentPipeline().settings).multiple) { + for (var target : result.targets) { + pointMap = new HashMap<>(); + pointMap.put("pitch", target.pitch); + pointMap.put("yaw", target.yaw); + pointMap.put("area", target.area); + pointMap.put("pose", target.cameraRelativePose); + webTargets.add(pointMap); + } + } else { + pointMap.put("pitch", bestTarget.pitch); + pointMap.put("yaw", bestTarget.yaw); + pointMap.put("area", bestTarget.area); + pointMap.put("pose", bestTarget.cameraRelativePose); + webTargets.add(pointMap); + } + center.add(bestTarget.minAreaRect.center.x); + center.add(bestTarget.minAreaRect.center.y); + + } } else { + pointMap.put("pitch", null); + pointMap.put("yaw", null); + pointMap.put("area", null); + pointMap.put("pose", new Pose2d()); + webTargets.add(pointMap); center.add(null); center.add(null); - calculated.put("pitch", null); - calculated.put("yaw", null); - calculated.put("area", null); } - } else { - center.add(null); - center.add(null); - calculated.put("pitch", null); - calculated.put("yaw", null); - calculated.put("area", null); + + point.put("fps", visionRunnable.fps); + point.put("targets", webTargets); + point.put("rawPoint", center); + WebSend.put("point", point); + SocketHandler.broadcastMessage(WebSend); } - point.put("fps", visionRunnable.fps); - point.put("calculated", calculated); - point.put("rawPoint", center); - WebSend.put("point", point); - SocketHandler.broadcastMessage(WebSend); } } private void updateNetworkTableData(CVPipelineResult data) { ntValidEntry.setBoolean(data.hasTarget); - if(data.hasTarget && !(data instanceof DriverVisionPipeline.DriverPipelineResult)) { - if(data instanceof CVPipeline2d.CVPipeline2dResult) { + if (data.hasTarget && !(data instanceof DriverVisionPipeline.DriverPipelineResult)) { + if (data instanceof StandardCVPipeline.StandardCVPipelineResult) { //noinspection unchecked - List targets = (List) data.targets; - ntTimeStampEntry.setDouble(data.imageTimestamp); + List targets = (List) data.targets; + ntLatencyEntry.setDouble(MathHandler.roundTo(data.processTime * 1e-6, 3)); ntPitchEntry.setDouble(targets.get(0).pitch); ntYawEntry.setDouble(targets.get(0).yaw); ntAreaEntry.setDouble(targets.get(0).area); try { - ntAuxListEntry.setString(objectMapper.writeValueAsString(targets)); + ntAuxListEntry.setString(objectMapper.writeValueAsString(targets.stream() + .map(it -> List.of(it.pitch, it.yaw, it.area, it.cameraRelativePose)) + .collect(Collectors.toList()))); + + // TODO: (2.1) 3d stuff... + ntPoseEntry.setString(objectMapper.writeValueAsString(targets.stream().map(target -> target.cameraRelativePose).collect(Collectors.toList()))); } catch (JsonProcessingException e) { e.printStackTrace(); } - } else if (data instanceof CVPipeline3d.CVPipeline3dResult) { - // TODO: (2.1) 3d stuff... + } else { + ntPitchEntry.setDouble(0.0); + ntYawEntry.setDouble(0.0); + ntAreaEntry.setDouble(0.0); + ntLatencyEntry.setDouble(0.0); + ntAuxListEntry.setString(""); } - } else { - ntPitchEntry.setDouble(0.0); - ntYawEntry.setDouble(0.0); - ntAreaEntry.setDouble(0.0); - ntTimeStampEntry.setDouble(0.0); - ntAuxListEntry.setString(""); } + tableInstance.flush(); + } public void setVideoMode(VideoMode newMode) { @@ -229,6 +275,27 @@ public class VisionProcess { return pipelineManager.driverModePipeline.settings; } + public void addCalibration(CameraCalibrationConfig cal) { + cameraCapture.addCalibrationData(cal); + System.out.println("saving to file"); + fileConfig.saveCalibration(cameraCapture.getConfig()); + } + + public void setIs3d(Boolean value) { + var settings = pipelineManager.getCurrentPipeline().settings; + if (settings instanceof StandardCVPipelineSettings) { + ((StandardCVPipelineSettings) settings).is3D = value; + } + } + + public boolean getIs3d() { + var settings = pipelineManager.getCurrentPipeline().settings; + if (settings instanceof StandardCVPipelineSettings) { + return ((StandardCVPipelineSettings) settings).is3D; + } + return false; + } + /** * VisionProcessRunnable will process images as quickly as possible */ @@ -240,14 +307,22 @@ public class VisionProcess { @Override public void run() { var lastUpdateTimeNanos = System.nanoTime(); - while(!Thread.interrupted()) { + var lastStreamTimeMs = System.currentTimeMillis(); + while (!Thread.interrupted()) { // blocking call, will block until camera has a new frame. Pair camData = cameraCapture.getFrame(); Mat camFrame = camData.getLeft(); if (camFrame.cols() > 0 && camFrame.rows() > 0) { - CVPipelineResult result = pipelineManager.getCurrentPipeline().runPipeline(camFrame); + CVPipelineResult result = null; + try { + result = pipelineManager.getCurrentPipeline().runPipeline(camFrame); + } catch (Exception e) { + System.err.println("Exception in vision process " + getCamera().getProperties().getNickname() + "!"); + e.printStackTrace(); + } + camFrame.release(); if (result != null) { @@ -259,8 +334,16 @@ public class VisionProcess { } try { - streamFrameQueue.clear(); - streamFrameQueue.add(lastPipelineResult.outputMat); +// streamFrameQueue.clear(); +// streamFrameQueue.add(lastPipelineResult.outputMat); + var currentTime = System.currentTimeMillis(); + if((currentTime - lastStreamTimeMs)/1000d > 1.0 / 30.0) { + cameraStreamer.runStream(lastPipelineResult.outputMat); +// System.out.println("Ran stream in " + (System.currentTimeMillis() - currentTime) + "ms!"); + lastStreamTimeMs = currentTime; + lastPipelineResult.outputMat.release(); + } + } catch (Exception e) { Debug.printInfo("Vision running faster than stream."); } @@ -274,7 +357,7 @@ public class VisionProcess { double getAverageFPS() { var temp = 0.0; - for(int i = 0; i < 7; i++) { + for (int i = 0; i < 7; i++) { temp += fpsAveragingBuffer.get(i); } temp /= 7.0; @@ -290,7 +373,7 @@ public class VisionProcess { private CameraStreamerRunnable(int cameraFPS, CameraStreamer streamer) { // add 2 FPS to allow for a bit of overhead - super(1000L/(cameraFPS + 2)); + super(1000L / (cameraFPS + 2)); this.streamer = streamer; } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java index 949e0da77..a59a4b2cd 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraCapture.java @@ -1,9 +1,12 @@ package com.chameleonvision.vision.camera; +import com.chameleonvision.config.CameraCalibrationConfig; import com.chameleonvision.vision.image.CaptureProperties; import com.chameleonvision.vision.image.ImageCapture; import edu.wpi.cscore.VideoMode; +import java.util.List; + public interface CameraCapture extends ImageCapture { CaptureProperties getProperties(); @@ -39,4 +42,7 @@ public interface CameraCapture extends ImageCapture { * @param gain the new gain to set the camera to */ void setGain(int gain); + + CameraCalibrationConfig getCurrentCalibrationData(); + List getAllCalibrationData(); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java index b36a0ad60..b3c9a95bf 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/CameraStreamer.java @@ -14,13 +14,14 @@ import org.opencv.imgproc.Imgproc; public class CameraStreamer { private final CameraCapture cameraCapture; private final String name; - private StreamDivisor divisor = StreamDivisor.NONE; + private StreamDivisor divisor; private CvSource cvSource; private final Object streamBufferLock = new Object(); private Mat streamBuffer = new Mat(); private Size size; - public CameraStreamer(CameraCapture cameraCapture, String name) { + public CameraStreamer(CameraCapture cameraCapture, String name,StreamDivisor div) { + this.divisor = div; this.cameraCapture = cameraCapture; this.name = name; this.cvSource = CameraServer.getInstance().putVideo(name, @@ -35,28 +36,33 @@ public class CameraStreamer { } public void setDivisor(StreamDivisor newDivisor, boolean updateUI) { - if (divisor != newDivisor) { - this.divisor = newDivisor; - var camValues = cameraCapture.getProperties(); - var newWidth = camValues.getStaticProperties().imageWidth / newDivisor.value; - var newHeight = camValues.getStaticProperties().imageHeight / newDivisor.value; - this.size = new Size(newWidth, newHeight); - synchronized (streamBufferLock) { - this.streamBuffer = new Mat(newWidth, newHeight, CvType.CV_8UC3); - this.cvSource = CameraServer.getInstance().putVideo(this.name, - cameraCapture.getProperties().getStaticProperties().imageWidth / divisor.value, - cameraCapture.getProperties().getStaticProperties().imageHeight / divisor.value); - } - if (updateUI) { - SocketHandler.sendFullSettings(); - } + this.divisor = newDivisor; + var camValues = cameraCapture.getProperties(); + var newWidth = camValues.getStaticProperties().imageWidth / newDivisor.value; + var newHeight = camValues.getStaticProperties().imageHeight / newDivisor.value; + this.size = new Size(newWidth, newHeight); + synchronized (streamBufferLock) { + this.streamBuffer = new Mat(newWidth, newHeight, CvType.CV_8UC3); + VideoMode oldVideoMode = cvSource.getVideoMode(); + cvSource.setVideoMode(new VideoMode(oldVideoMode.pixelFormat, + cameraCapture.getProperties().getStaticProperties().imageWidth / divisor.value, + cameraCapture.getProperties().getStaticProperties().imageHeight / divisor.value, + oldVideoMode.fps)); } + if (updateUI) { + SocketHandler.sendFullSettings(); + } + } public StreamDivisor getDivisor() { return divisor; } + public void recalculateDivision() { + setDivisor(this.divisor, false); + } + public void setNewVideoMode(VideoMode newVideoMode) { // Trick to update cvSource and streamBuffer to the new resolution // Must change the cameraProcess resolution first diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java index f8b229c6d..129558ba0 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCameraCapture.java @@ -1,7 +1,7 @@ package com.chameleonvision.vision.camera; -import com.chameleonvision.config.CameraJsonConfig; -import com.chameleonvision.vision.image.CaptureProperties; +import com.chameleonvision.config.CameraCalibrationConfig; +import com.chameleonvision.config.FullCameraConfiguration; import edu.wpi.cscore.CvSink; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.VideoException; @@ -9,14 +9,23 @@ import edu.wpi.cscore.VideoMode; import edu.wpi.first.cameraserver.CameraServer; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgcodecs.Imgcodecs; + +import java.util.ArrayList; +import java.util.List; public class USBCameraCapture implements CameraCapture { private final UsbCamera baseCamera; private final CvSink cvSink; + private List calibrationList; private Mat imageBuffer = new Mat(); private USBCaptureProperties properties; - public USBCameraCapture(CameraJsonConfig config) { + public USBCameraCapture(FullCameraConfiguration fullCameraConfiguration) { + var config = fullCameraConfiguration.cameraConfig; + this.calibrationList = new ArrayList<>(); //fullCameraConfiguration.calibration; + calibrationList.addAll(fullCameraConfiguration.calibration); baseCamera = new UsbCamera(config.name, config.path); cvSink = CameraServer.getInstance().getVideo(baseCamera); properties = new USBCaptureProperties(baseCamera, config); @@ -25,6 +34,24 @@ public class USBCameraCapture implements CameraCapture { setVideoMode(videoMode); } + public CameraCalibrationConfig getCalibration(Size size) { + for(var calibration: calibrationList) { + if(calibration.resolution.equals(size)) return calibration; + } + return null; + } + + public CameraCalibrationConfig getCalibration(VideoMode mode) { + return getCalibration(new Size(mode.width, mode.height)); + } + + public void addCalibrationData(CameraCalibrationConfig newConfig) { + calibrationList.add(newConfig); + } + + public List getConfig() { + return calibrationList; + } @Override public USBCaptureProperties getProperties() { @@ -91,4 +118,14 @@ public class USBCameraCapture implements CameraCapture { } } } + + @Override + public CameraCalibrationConfig getCurrentCalibrationData() { + return getCalibration(getCurrentVideoMode()); + } + + @Override + public List getAllCalibrationData() { + return calibrationList; + } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCaptureProperties.java b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCaptureProperties.java index b08d6701b..2abbef913 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCaptureProperties.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/camera/USBCaptureProperties.java @@ -5,6 +5,7 @@ import com.chameleonvision.util.Platform; import com.chameleonvision.vision.image.CaptureProperties; import edu.wpi.cscore.UsbCamera; import edu.wpi.cscore.VideoMode; +import edu.wpi.first.wpilibj.geometry.Rotation2d; import java.util.Arrays; import java.util.List; @@ -12,7 +13,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -public class USBCaptureProperties extends com.chameleonvision.vision.image.CaptureProperties { +public class USBCaptureProperties extends CaptureProperties { public static final double DEFAULT_FOV = 70; private static final int DEFAULT_EXPOSURE = 50; private static final int DEFAULT_BRIGHTNESS = 50; @@ -100,10 +101,16 @@ public class USBCaptureProperties extends com.chameleonvision.vision.image.Captu return videoModes; } + public VideoMode getVideoMode(int index){ + return videoModes.get(index); + } + public VideoMode getCurrentVideoMode() { return staticProperties.mode; } public int getCurrentVideoModeIndex(){ return getVideoModes().indexOf(getCurrentVideoMode()); } + + } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/enums/ImageRotationMode.java b/chameleon-server/src/main/java/com/chameleonvision/vision/enums/ImageRotationMode.java index c7920b46d..b22150aab 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/enums/ImageRotationMode.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/enums/ImageRotationMode.java @@ -13,4 +13,6 @@ public enum ImageRotationMode { ImageRotationMode(int value) { this.value = value; } + + public boolean isRotated(){return this.value==DEG_90.value||this.value==DEG_270.value;} } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java b/chameleon-server/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java index 8717bb88d..c2e2374bc 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/image/CaptureProperties.java @@ -2,11 +2,13 @@ package com.chameleonvision.vision.image; import com.chameleonvision.vision.camera.CaptureStaticProperties; import edu.wpi.cscore.VideoMode; +import edu.wpi.first.wpilibj.geometry.Rotation2d; import org.opencv.core.Mat; public class CaptureProperties { protected CaptureStaticProperties staticProperties; + private Rotation2d tilt = new Rotation2d(); protected CaptureProperties() { } @@ -14,8 +16,16 @@ public class CaptureProperties { public CaptureProperties(VideoMode videoMode, double fov) { staticProperties = new CaptureStaticProperties(videoMode, fov); } - + public void setStaticProperties(CaptureStaticProperties staticProperties) {this.staticProperties = staticProperties;} public CaptureStaticProperties getStaticProperties() { return staticProperties; } + + public Rotation2d getTilt() { + return tilt; + } + + public void setTilt(Rotation2d tilt) { + this.tilt = tilt; + } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java b/chameleon-server/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java index 5de9999b7..820a9097d 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/image/StaticImageCapture.java @@ -1,5 +1,6 @@ package com.chameleonvision.vision.image; +import com.chameleonvision.config.CameraCalibrationConfig; import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.vision.camera.USBCaptureProperties; import edu.wpi.cscore.VideoMode; @@ -9,6 +10,7 @@ import org.opencv.imgcodecs.Imgcodecs; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; public class StaticImageCapture implements CameraCapture { @@ -73,4 +75,14 @@ public class StaticImageCapture implements CameraCapture { public void setGain(int gain) { // do nothing } + + @Override + public CameraCalibrationConfig getCurrentCalibrationData() { + return null; + } + + @Override + public List getAllCalibrationData() { + return null; + } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java index 092939ace..a51073778 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline.java @@ -9,7 +9,7 @@ import org.opencv.core.Mat; */ public abstract class CVPipeline { protected Mat outputMat = new Mat(); - CameraCapture cameraCapture; + protected CameraCapture cameraCapture; public S settings; protected CVPipeline(S settings) { @@ -29,8 +29,4 @@ public abstract class CVPipeline { - - - protected CVPipeline3d(CVPipeline3dSettings settings) { - super(settings); - } - - CVPipeline3d() { - super(new CVPipeline3dSettings()); - } - - @Override - public CVPipeline3dResult runPipeline(Mat inputMat) { - return null; - } - - - public static class CVPipeline3dResult extends CVPipelineResult { - public CVPipeline3dResult(List targets, Mat outputMat, long processTime) { - super(targets, outputMat, processTime); - } - } - - public static class Target3d extends CVPipeline2d.Target2d { - // TODO: (2.1) Define 3d-specific target data - } -} diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline3dSettings.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline3dSettings.java deleted file mode 100644 index 342659ca1..000000000 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline3dSettings.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.chameleonvision.vision.pipeline; - -public class CVPipeline3dSettings extends CVPipeline2dSettings { - // TODO: (2.1) define 3d-specific pipeline settings - // add 3d-specific property to ensure serializing/deserializing works - public boolean placeholder = false; -} diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Pipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/Pipe.java similarity index 84% rename from chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Pipe.java rename to chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/Pipe.java index d2458a6c7..8fd8104d0 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Pipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/Pipe.java @@ -1,4 +1,4 @@ -package com.chameleonvision.vision.pipeline.pipes; +package com.chameleonvision.vision.pipeline; import org.apache.commons.lang3.tuple.Pair; diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/PipelineManager.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/PipelineManager.java index 7e6a942a1..11f6acafb 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/PipelineManager.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/PipelineManager.java @@ -1,10 +1,13 @@ package com.chameleonvision.vision.pipeline; +import com.chameleonvision.Exceptions.DuplicatedKeyException; import com.chameleonvision.config.CameraConfig; import com.chameleonvision.config.ConfigManager; import com.chameleonvision.vision.VisionManager; import com.chameleonvision.vision.VisionProcess; +import com.chameleonvision.vision.pipeline.impl.*; import com.chameleonvision.web.SocketHandler; +import edu.wpi.cscore.VideoMode; import edu.wpi.first.networktables.NetworkTableEntry; import java.util.Comparator; @@ -16,10 +19,12 @@ import java.util.List; public class PipelineManager { private static final int DRIVERMODE_INDEX = -1; + private static final int CAL_3D_INDEX = -2; public final LinkedList pipelines = new LinkedList<>(); public final CVPipeline driverModePipeline = new DriverVisionPipeline(new CVPipelineSettings()); + public final Calibrate3dPipeline calib3dPipe = new Calibrate3dPipeline(new StandardCVPipelineSettings()); private final VisionProcess parentProcess; private int lastPipelineIndex; @@ -29,7 +34,7 @@ public class PipelineManager { public PipelineManager(VisionProcess visionProcess, List loadedPipelineSettings) { parentProcess = visionProcess; if (loadedPipelineSettings == null || loadedPipelineSettings.size() == 0) { - pipelines.add(new CVPipeline2d("New Pipeline")); + pipelines.add(new StandardCVPipeline("New Pipeline")); } else { for (CVPipelineSettings setting : loadedPipelineSettings) { addInternalPipeline(setting); @@ -71,10 +76,8 @@ public class PipelineManager { } private void addInternalPipeline(CVPipelineSettings setting) { - if (setting instanceof CVPipeline3dSettings) { - pipelines.add(new CVPipeline3d((CVPipeline3dSettings) setting)); - } else if (setting instanceof CVPipeline2dSettings) { - pipelines.add(new CVPipeline2d((CVPipeline2dSettings) setting)); + if (setting instanceof StandardCVPipelineSettings) { + pipelines.add(new StandardCVPipeline((StandardCVPipelineSettings) setting)); } else { System.out.println("Non 2D/3D pipelines not supported!"); } @@ -82,11 +85,18 @@ public class PipelineManager { } public void setDriverMode(boolean driverMode) { - if (driverMode) { - setCurrentPipeline(DRIVERMODE_INDEX); - } else { - setCurrentPipeline(lastPipelineIndex); - } + if (driverMode) setCurrentPipeline(DRIVERMODE_INDEX); + else setCurrentPipeline(lastPipelineIndex); + } + + public void setCalibrationMode(boolean calibrationMode) { + setCurrentPipeline((calibrationMode ? CAL_3D_INDEX : lastPipelineIndex)); + } + + public void enableCalibrationMode(VideoMode mode) { + parentProcess.setVideoMode(mode); + calib3dPipe.setVideoMode(mode); + setCalibrationMode(true); } public boolean getDriverMode() { @@ -98,32 +108,45 @@ public class PipelineManager { } public CVPipeline getCurrentPipeline() { - if (currentPipelineIndex <= DRIVERMODE_INDEX) { + if (currentPipelineIndex == DRIVERMODE_INDEX) { return driverModePipeline; + } else if (currentPipelineIndex <= CAL_3D_INDEX) { + return calib3dPipe; } else { return pipelines.get(currentPipelineIndex); } } public void setCurrentPipeline(int index) { - CVPipeline newPipeline; + CVPipeline newPipeline=null; if (index == DRIVERMODE_INDEX) { newPipeline = driverModePipeline; - // if we're changing into driver mode, try to set the nt entry to frue + // if we're changing into driver mode, try to set the nt entry to true + parentProcess.setDriverModeEntry(true); + } else if (index == CAL_3D_INDEX) { parentProcess.setDriverModeEntry(true); - } else { - newPipeline = pipelines.get(index); - // if we're switching out of driver mode, try to set the nt entry to false - parentProcess.setDriverModeEntry(false); + newPipeline = calib3dPipe; + } else { + if (index < pipelines.size()&&index>=0) { + newPipeline = pipelines.get(index); + + // if we're switching out of driver mode, try to set the nt entry to false + parentProcess.setDriverModeEntry(false); + } + else + { + //TODO alert/warn user that pipeline doesnt exsits + System.err.println("Index is out of bounds"); + } } if (newPipeline != null) { lastPipelineIndex = currentPipelineIndex; currentPipelineIndex = index; getCurrentPipeline().initPipeline(parentProcess.getCamera()); - if(ConfigManager.settings.currentCamera.equals(parentProcess.getCamera().getProperties().name)) { + if (ConfigManager.settings.currentCamera.equals(parentProcess.getCamera().getProperties().name)) { ConfigManager.settings.currentPipeline = currentPipelineIndex; HashMap pipeChange = new HashMap<>(); @@ -136,7 +159,9 @@ public class PipelineManager { } } newPipeline.initPipeline(parentProcess.getCamera()); - if(ntIndexEntry != null) { + if (parentProcess.cameraStreamer != null) + parentProcess.cameraStreamer.setDivisor(newPipeline.settings.streamDivisor, true); + if (ntIndexEntry != null) { ntIndexEntry.setDouble(index); } } @@ -153,13 +178,9 @@ public class PipelineManager { savePipelineConfig(pipeline.settings); } - public void addNewPipeline(boolean is3D) { - CVPipeline newPipeline; - if (!is3D) { - newPipeline = new CVPipeline2d(); - } else { - newPipeline = new CVPipeline3d(); - } + public void addNewPipeline(String piplineName) { + StandardCVPipeline newPipeline = new StandardCVPipeline(); + newPipeline.settings.nickname = piplineName; newPipeline.settings.index = pipelines.size(); addPipeline(newPipeline); } @@ -168,19 +189,22 @@ public class PipelineManager { return pipelines.get(index); } - public void duplicatePipeline(CVPipelineSettings pipeline) { + public void duplicatePipeline(CVPipelineSettings pipeline) throws DuplicatedKeyException { duplicatePipeline(pipeline, parentProcess); } - public void duplicatePipeline(CVPipelineSettings pipeline, VisionProcess destinationProcess) { + public void duplicatePipeline(CVPipelineSettings pipeline, VisionProcess destinationProcess) throws DuplicatedKeyException { pipeline.index = destinationProcess.pipelineManager.pipelines.size(); pipeline.nickname += "(Copy)"; - destinationProcess.pipelineManager.addPipeline(pipeline); + if (destinationProcess.pipelineManager.pipelines.stream().anyMatch(c -> c.settings.nickname.equals(pipeline.nickname))){ + throw new DuplicatedKeyException("key Already exists"); + } else{ + destinationProcess.pipelineManager.addPipeline(pipeline); + } } public void renameCurrentPipeline(String newName) { CVPipelineSettings settings = getCurrentPipeline().settings; - settings.nickname = newName; renamePipelineConfig(settings, newName); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/Calibrate3dPipeline.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/Calibrate3dPipeline.java new file mode 100644 index 000000000..fbba9856c --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/Calibrate3dPipeline.java @@ -0,0 +1,167 @@ +package com.chameleonvision.vision.pipeline.impl; + +import com.chameleonvision.config.CameraCalibrationConfig; +import com.chameleonvision.config.ConfigManager; +import com.chameleonvision.vision.VisionManager; +import com.chameleonvision.vision.camera.CameraCapture; +import com.chameleonvision.vision.pipeline.CVPipeline; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.wpi.cscore.VideoMode; +import edu.wpi.first.wpilibj.util.Units; +import org.opencv.calib3d.Calib3d; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; + +import java.util.ArrayList; +import java.util.List; + +public class Calibrate3dPipeline extends CVPipeline { + + private int checkerboardSquaresHigh = 7; + private int checkerboardSquaresWide = 7; + private MatOfPoint3f objP_ORIG; + + private MatOfPoint3f objP;// new MatOfPoint3f(checkerboardSquaresHigh + checkerboardSquaresWide, 3);//(checkerboardSquaresWide * checkerboardSquaresHigh, 3); + private Size patternSize = new Size(checkerboardSquaresHigh, checkerboardSquaresWide); + private Size imageSize; + double checkerboardSquareSize = 1; // inches! + private MatOfPoint2f calibrationOutput = new MatOfPoint2f(); + private List objpoints = new ArrayList<>(); + private List imgpoints = new ArrayList<>(); + + public static double checkerboardSquareSizeUnits = Units.inchesToMeters(1.0); + + public static final int MIN_COUNT = 15; + private VideoMode calibrationMode; + private final Size windowSize = new Size(11, 11); + private final Size zeroZone = new Size(-1, -1); + private TermCriteria criteria = new TermCriteria(3, 30, 0.001); //(Imgproc.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001) + + private int captureCount = 0; + private boolean wantsSnapshot = false; + private double squareSizeInches; + + public Calibrate3dPipeline(StandardCVPipelineSettings settings) { + super(settings); + + objP_ORIG = new MatOfPoint3f(); + objP = new MatOfPoint3f(); + + for(int i = 0; i < checkerboardSquaresHigh * checkerboardSquaresWide; i++) { + objP_ORIG.push_back(new MatOfPoint3f(new Point3(i / checkerboardSquaresWide, i % checkerboardSquaresHigh, 0.0f))); + } + + setSquareSize(checkerboardSquareSizeUnits); + + objpoints.forEach(Mat::release); + imgpoints.forEach(Mat::release); + objpoints.clear(); + imgpoints.clear(); + } + + public void setSquareSize(double size) { + this.squareSizeInches = size; + } + + public void takeSnapshot() { + wantsSnapshot = true; + } + + public boolean hasEnoughSnapshots() { + return captureCount >= MIN_COUNT - 1; + } + + @Override + public DriverVisionPipeline.DriverPipelineResult runPipeline(Mat inputMat) { + + // look for checkerboard + Imgproc.cvtColor(inputMat, inputMat, Imgproc.COLOR_BGR2GRAY); + var checkerboardFound = Calib3d.findChessboardCorners(inputMat, patternSize, calibrationOutput); + + + + if(!checkerboardFound) { + Imgproc.cvtColor(inputMat, inputMat, Imgproc.COLOR_GRAY2BGR); + + return new DriverVisionPipeline.DriverPipelineResult(null, inputMat, 0); + } + +// System.out.println("[SolvePNP] checkerboard found!!"); + + // cool we found a checkerboard + // do corner subpixel + Imgproc.cornerSubPix(inputMat, calibrationOutput, windowSize, zeroZone, criteria); + + // convert back to BGR + Imgproc.cvtColor(inputMat, inputMat, Imgproc.COLOR_GRAY2BGR); + // draw the chessboard + Calib3d.drawChessboardCorners(inputMat, patternSize, calibrationOutput, true); + + if(wantsSnapshot) { + this.imageSize = new Size(inputMat.width(), inputMat.height()); + + var mat = new MatOfPoint3f(); + calibrationOutput.copyTo(mat); + this.objpoints.add(objP); + imgpoints.add(mat); + captureCount++; + wantsSnapshot = false; + } + + imageSize = new Size(inputMat.width(), inputMat.height()); + + return new DriverVisionPipeline.DriverPipelineResult(null, inputMat, 0); + } + + @Override + public void initPipeline(CameraCapture camera) { + super.initPipeline(camera); + objpoints.clear(); + imgpoints.clear(); + captureCount = 0; + } + + public boolean tryCalibration() { + if (!hasEnoughSnapshots()) return false; + + Mat cameraMatrix = new Mat(); + Mat distortionCoeffs = new Mat(); + List rvecs = new ArrayList<>(); + List tvecs = new ArrayList<>(); + + try { + Calib3d.calibrateCamera(objpoints, imgpoints, imageSize, cameraMatrix, distortionCoeffs, rvecs, tvecs); + } catch(Exception e) { + System.err.println("Camera calibration failed!"); + initPipeline(cameraCapture); + return false; + } + + VideoMode currentVidMode = cameraCapture.getCurrentVideoMode(); + Size resolution = new Size(currentVidMode.width, currentVidMode.height); + CameraCalibrationConfig cal = new CameraCalibrationConfig(resolution, cameraMatrix, distortionCoeffs, squareSizeInches); + + VisionManager.getCurrentUIVisionProcess().addCalibration(cal); + + try { + System.out.printf("CALIBRATION SUCCESS! camMatrix: \n%s\ndistortionCoeffs:\n%s\n", + new ObjectMapper().writeValueAsString(cal.cameraMatrix), new ObjectMapper().writeValueAsString(cal.distortionCoeffs)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + + + ConfigManager.saveGeneralSettings(); + + return true; + } + + public void setVideoMode(VideoMode mode){ + this.calibrationMode = mode; + } + + public int getSnapshotCount() { + return captureCount + 1; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/DriverVisionPipeline.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/DriverVisionPipeline.java similarity index 57% rename from chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/DriverVisionPipeline.java rename to chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/DriverVisionPipeline.java index e5cc03baa..1dcff6a30 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/DriverVisionPipeline.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/DriverVisionPipeline.java @@ -1,8 +1,12 @@ -package com.chameleonvision.vision.pipeline; +package com.chameleonvision.vision.pipeline.impl; import com.chameleonvision.util.MemoryManager; import com.chameleonvision.vision.camera.CameraCapture; -import com.chameleonvision.vision.pipeline.pipes.Draw2dContoursPipe; +import com.chameleonvision.vision.enums.CalibrationMode; +import com.chameleonvision.vision.pipeline.CVPipeline; +import com.chameleonvision.vision.pipeline.CVPipelineResult; +import com.chameleonvision.vision.pipeline.CVPipelineSettings; +import com.chameleonvision.vision.pipeline.pipes.Draw2dCrosshairPipe; import com.chameleonvision.vision.pipeline.pipes.RotateFlipPipe; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; @@ -10,14 +14,13 @@ import org.opencv.core.RotatedRect; import java.util.List; -import static com.chameleonvision.vision.pipeline.DriverVisionPipeline.DriverPipelineResult; +import static com.chameleonvision.vision.pipeline.impl.DriverVisionPipeline.DriverPipelineResult; public class DriverVisionPipeline extends CVPipeline { private RotateFlipPipe rotateFlipPipe; - private Draw2dContoursPipe draw2dContoursPipe; - private Draw2dContoursPipe.Draw2dContoursSettings draw2dContoursSettings = new Draw2dContoursPipe.Draw2dContoursSettings(); - private final List blankList = List.of(); + private Draw2dCrosshairPipe drawCrosshairPipe; + private Draw2dCrosshairPipe.Draw2dCrosshairPipeSettings crosshairPipeSettings = new Draw2dCrosshairPipe.Draw2dCrosshairPipeSettings(); private final MemoryManager memoryManager = new MemoryManager(200, 20000); @@ -30,22 +33,20 @@ public class DriverVisionPipeline extends CVPipeline rotateFlipResult = rotateFlipPipe.run(inputMat); - Pair draw2dContoursResult = draw2dContoursPipe.run(Pair.of(rotateFlipResult.getLeft(), blankList)); - + Pair draw2dCrosshairResult = drawCrosshairPipe.run(Pair.of(rotateFlipResult.getLeft(),null)); memoryManager.run(); - return new DriverPipelineResult(null, draw2dContoursResult.getLeft(), 0); + return new DriverPipelineResult(null, draw2dCrosshairResult.getLeft(), 0); } public static class DriverPipelineResult extends CVPipelineResult { diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/StandardCVPipeline.java similarity index 62% rename from chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java rename to chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/StandardCVPipeline.java index dd211a5a1..e5407baac 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2d.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/StandardCVPipeline.java @@ -1,19 +1,22 @@ -package com.chameleonvision.vision.pipeline; +package com.chameleonvision.vision.pipeline.impl; import com.chameleonvision.Main; import com.chameleonvision.util.MemoryManager; import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.vision.camera.CaptureStaticProperties; +import com.chameleonvision.vision.pipeline.CVPipeline; +import com.chameleonvision.vision.pipeline.CVPipelineResult; import com.chameleonvision.vision.pipeline.pipes.*; +import edu.wpi.first.wpilibj.geometry.Pose2d; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.*; import java.util.List; -import static com.chameleonvision.vision.pipeline.CVPipeline2d.*; +import static com.chameleonvision.vision.pipeline.impl.StandardCVPipeline.*; @SuppressWarnings("WeakerAccess") -public class CVPipeline2d extends CVPipeline { +public class StandardCVPipeline extends CVPipeline { private Mat rawCameraMat = new Mat(); @@ -29,21 +32,25 @@ public class CVPipeline2d extends CVPipeline rotateFlipResult = rotateFlipPipe.run(inputMat); totalPipelineTimeNanos += rotateFlipResult.getRight(); + inputMat.copyTo(rawCameraMat); + // Pair blurResult = blurPipe.run(rotateFlipResult.getLeft()); // totalPipelineTimeNanos += blurResult.getRight(); @@ -143,22 +164,47 @@ public class CVPipeline2d extends CVPipeline, Long> speckleRejectResult = speckleRejectPipe.run(filterContoursResult.getLeft()); totalPipelineTimeNanos += speckleRejectResult.getRight(); - Pair, Long> groupContoursResult = groupContoursPipe.run(speckleRejectResult.getLeft()); + Pair, Long> groupContoursResult = groupContoursPipe.run(speckleRejectResult.getLeft()); totalPipelineTimeNanos += groupContoursResult.getRight(); - Pair, Long> sortContoursResult = sortContoursPipe.run(groupContoursResult.getLeft()); + Pair, Long> sortContoursResult = sortContoursPipe.run(groupContoursResult.getLeft()); totalPipelineTimeNanos += sortContoursResult.getRight(); - Pair, Long> collect2dTargetsResult = collect2dTargetsPipe.run(Pair.of(sortContoursResult.getLeft(), camProps)); + Pair, Long> collect2dTargetsResult = collect2dTargetsPipe.run(Pair.of(sortContoursResult.getLeft(), camProps)); totalPipelineTimeNanos += collect2dTargetsResult.getRight(); // takes pair of (Mat of original camera image (8UC3), Mat of HSV thresholded image(8UC1)) - Pair outputMatResult = outputMatPipe.run(Pair.of(rotateFlipResult.getLeft(), hsvResult.getLeft())); + Pair outputMatResult = outputMatPipe.run(Pair.of(rawCameraMat, hsvResult.getLeft())); totalPipelineTimeNanos += outputMatResult.getRight(); + Pair result; + + if(!settings.is3D) { + // takes pair of (Mat to draw on, List of sorted contours) + result = draw2dContoursPipe.run(Pair.of(outputMatResult.getLeft(), sortContoursResult.getLeft())); + totalPipelineTimeNanos += result.getRight(); + } else { + result = outputMatResult; + } + // takes pair of (Mat to draw on, List of sorted contours) - Pair draw2dContoursResult = draw2dContoursPipe.run(Pair.of(outputMatResult.getLeft(), sortContoursResult.getLeft())); - totalPipelineTimeNanos += draw2dContoursResult.getRight(); + Pair draw2dCrosshairResult = draw2dCrosshairPipe.run(Pair.of(result.getLeft(),collect2dTargetsResult.getLeft())); + totalPipelineTimeNanos += draw2dCrosshairResult.getRight(); + + Mat outputMat; + + if(settings.is3D) { + // once we've sorted our targets, perform solvePNP. The number of "best targets" is limited by the above pipe + Pair, Long> solvePNPResult = solvePNPPipe.run(collect2dTargetsResult.getLeft()); + totalPipelineTimeNanos += solvePNPResult.getRight(); + + Pair draw3dContoursResult = drawSolvePNPPipe.run(Pair.of(outputMatResult.getLeft(), solvePNPResult.getLeft())); + totalPipelineTimeNanos += draw3dContoursResult.getRight(); + + outputMat = draw3dContoursResult.getLeft(); + } else { + outputMat = draw2dCrosshairResult.getLeft(); + } if (Main.testMode) { pipelineTimeString += String.format("PipeInit: %.2fms, ", pipeInitTimeNanos / 1000000.0); @@ -173,7 +219,8 @@ public class CVPipeline2d extends CVPipeline { - public CVPipeline2dResult(List targets, Mat outputMat, long processTimeNanos) { + public static class StandardCVPipelineResult extends CVPipelineResult { + public StandardCVPipelineResult(List targets, Mat outputMat, long processTimeNanos) { super(targets, outputMat, processTimeNanos); } + + public void release() { + targets.forEach(TrackedTarget::release); + outputMat.release(); + } } - public static class Target2d { + public static class TrackedTarget { public double calibratedX = 0.0; public double calibratedY = 0.0; public double pitch = 0.0; public double yaw = 0.0; public double area = 0.0; - public RotatedRect rawPoint; + public RotatedRect minAreaRect; + + // 3d stuff + public Pose2d cameraRelativePose = new Pose2d(); + public Mat rVector = new Mat(); + public Mat tVector = new Mat(); + public MatOfPoint2f imageCornerPoints = new MatOfPoint2f(); + public Pair leftRightDualTargetPair = null; + public Pair leftRightRotatedRect = null; + + public void release() { + rVector.release(); + tVector.release(); + imageCornerPoints.release(); + } } + + } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2dSettings.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/StandardCVPipelineSettings.java similarity index 69% rename from chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2dSettings.java rename to chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/StandardCVPipelineSettings.java index 2f3be3ed8..a6db1e5fc 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/CVPipeline2dSettings.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/impl/StandardCVPipelineSettings.java @@ -1,14 +1,21 @@ -package com.chameleonvision.vision.pipeline; +package com.chameleonvision.vision.pipeline.impl; import com.chameleonvision.vision.enums.CalibrationMode; import com.chameleonvision.vision.enums.SortMode; import com.chameleonvision.vision.enums.TargetGroup; import com.chameleonvision.vision.enums.TargetIntersection; +import com.chameleonvision.vision.pipeline.CVPipelineSettings; +import com.fasterxml.jackson.annotation.JsonIgnore; +import edu.wpi.first.wpilibj.util.Units; +import org.opencv.core.Mat; +import org.opencv.core.MatOfDouble; +import org.opencv.core.Point; +import org.opencv.core.Point3; import java.util.Arrays; import java.util.List; -public class CVPipeline2dSettings extends CVPipelineSettings { +public class StandardCVPipelineSettings extends CVPipelineSettings { public List hue = Arrays.asList(50, 180); public List saturation = Arrays.asList(50, 255); public List value = Arrays.asList(50, 255); @@ -27,4 +34,9 @@ public class CVPipeline2dSettings extends CVPipelineSettings { public CalibrationMode calibrationMode = CalibrationMode.None; public double dualTargetCalibrationM = 1; public double dualTargetCalibrationB = 0; + + // 3d stuff + public double targetWidth = 15.5, targetHeight = 6.0; + + public boolean is3D = false; } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java index a6ee819b5..95f99b7be 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/BlurPipe.java @@ -1,10 +1,8 @@ package com.chameleonvision.vision.pipeline.pipes; +import com.chameleonvision.vision.pipeline.Pipe; import org.apache.commons.lang3.tuple.Pair; -import org.opencv.core.CvException; import org.opencv.core.Mat; -import org.opencv.core.Size; -import org.opencv.imgproc.Imgproc; public class BlurPipe implements Pipe { diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java index 572e9e6e2..4e7bdc878 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Collect2dTargetsPipe.java @@ -1,45 +1,38 @@ package com.chameleonvision.vision.pipeline.pipes; import com.chameleonvision.vision.camera.CaptureStaticProperties; -import com.chameleonvision.vision.pipeline.CVPipeline2d; +import com.chameleonvision.vision.pipeline.Pipe; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; import com.chameleonvision.vision.enums.CalibrationMode; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.math3.util.FastMath; -import org.opencv.core.Mat; -import org.opencv.core.RotatedRect; import java.util.ArrayList; import java.util.List; -public class Collect2dTargetsPipe implements Pipe, CaptureStaticProperties>, List> { +public class Collect2dTargetsPipe implements Pipe, CaptureStaticProperties>, List> { + - private CalibrationMode calibrationMode; private CaptureStaticProperties camProps; + private CalibrationMode calibrationMode; private List calibrationPoint; private double calibrationM, calibrationB; + private List targets = new ArrayList<>(); - private List targets = new ArrayList<>(); - - public Collect2dTargetsPipe(CalibrationMode calibrationMode, List calibrationPoint, - double calibrationM, double calibrationB, CaptureStaticProperties camProps) { - this.calibrationMode = calibrationMode; - this.camProps = camProps; - this.calibrationPoint = calibrationPoint; - this.calibrationM = calibrationM; - this.calibrationB = calibrationB; + public Collect2dTargetsPipe(CalibrationMode calibrationMode, List calibrationPoint, double calibrationM, double calibrationB, CaptureStaticProperties camProps) { + setConfig(calibrationMode, calibrationPoint, calibrationM, calibrationB, camProps); } - public void setConfig(CalibrationMode calibrationMode, List calibrationPoint, - double calibrationM, double calibrationB, CaptureStaticProperties camProps) { + public void setConfig(CalibrationMode calibrationMode, List calibrationPoint, double calibrationM, double calibrationB, CaptureStaticProperties camProps) { this.calibrationMode = calibrationMode; - this.camProps = camProps; this.calibrationPoint = calibrationPoint; this.calibrationM = calibrationM; this.calibrationB = calibrationB; + this.camProps = camProps; } @Override - public Pair, Long> run(Pair, CaptureStaticProperties> inputPair) { + public Pair, Long> run(Pair, CaptureStaticProperties> inputPair) { long processStartNanos = System.nanoTime(); targets.clear(); @@ -47,27 +40,30 @@ public class Collect2dTargetsPipe implements Pipe, Captur var imageArea = inputPair.getRight().imageArea; if (input.size() > 0) { - for (RotatedRect r : input) { - CVPipeline2d.Target2d t = new CVPipeline2d.Target2d(); - t.rawPoint = r; - switch (calibrationMode) { + for (var t : input) { + switch (this.calibrationMode) { + case Single: + if(this.calibrationPoint.isEmpty()) + { + this.calibrationPoint.add(camProps.centerX); + this.calibrationPoint.add(camProps.centerY); + } + t.calibratedX = this.calibrationPoint.get(0).doubleValue(); + t.calibratedY = this.calibrationPoint.get(1).doubleValue(); + break; case None: t.calibratedX = camProps.centerX; t.calibratedY = camProps.centerY; break; - case Single: - t.calibratedX = calibrationPoint.get(0).doubleValue(); - t.calibratedY = calibrationPoint.get(1).doubleValue(); - break; case Dual: - t.calibratedX = (r.center.y - calibrationB) / calibrationM; - t.calibratedY = (r.center.x * calibrationM) + calibrationB; + t.calibratedX = (t.minAreaRect.center.y - this.calibrationB) / this.calibrationM; + t.calibratedY = (t.minAreaRect.center.x * this.calibrationM) + this.calibrationB; break; } - t.pitch = calculatePitch(r.center.y, t.calibratedY); - t.yaw = calculateYaw(r.center.x, t.calibratedX); - t.area = r.size.area() / imageArea; + t.pitch = calculatePitch(t.minAreaRect.center.y, t.calibratedY); + t.yaw = calculateYaw(t.minAreaRect.center.x, t.calibratedX); + t.area = t.minAreaRect.size.area() / imageArea; targets.add(t); } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java index af7655af6..9fdb6def9 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dContoursPipe.java @@ -2,6 +2,8 @@ package com.chameleonvision.vision.pipeline.pipes; import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.util.Helpers; +import com.chameleonvision.vision.pipeline.Pipe; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Point; import org.opencv.core.*; @@ -11,7 +13,7 @@ import java.awt.*; import java.util.ArrayList; import java.util.List; -public class Draw2dContoursPipe implements Pipe>, Mat> { +public class Draw2dContoursPipe implements Pipe>, Mat> { private final Draw2dContoursSettings settings; private CaptureStaticProperties camProps; @@ -37,10 +39,10 @@ public class Draw2dContoursPipe implements Pipe>, Ma } @Override - public Pair run(Pair> input) { + public Pair run(Pair> input) { long processStartNanos = System.nanoTime(); - if (settings.showCrosshair || settings.showCentroid || settings.showMaximumBox || settings.showRotatedBox) { + if (settings.showCentroid || settings.showMaximumBox || settings.showRotatedBox) { // input.getLeft().copyTo(processBuffer); // processBuffer = input.getLeft(); @@ -49,11 +51,13 @@ public class Draw2dContoursPipe implements Pipe>, Ma if (i != 0 && !settings.showMultiple){ break; } - RotatedRect r = input.getRight().get(i); + StandardCVPipeline.TrackedTarget target = input.getRight().get(i); + RotatedRect r = input.getRight().get(i).minAreaRect; if (r == null) continue; drawnContours.forEach(Mat::release); drawnContours.clear(); + drawnContours = new ArrayList<>(); r.points(vertices); contour.fromArray(vertices); @@ -77,34 +81,22 @@ public class Draw2dContoursPipe implements Pipe>, Ma } } - if (settings.showCrosshair) { - xMax.set(new double[] {camProps.centerX + 10, camProps.centerY}); - xMin.set(new double[] {camProps.centerX - 10, camProps.centerY}); - yMax.set(new double[] {camProps.centerX, camProps.centerY + 10}); - yMin.set(new double[] {camProps.centerX, camProps.centerY - 10}); - Imgproc.line(input.getLeft(), xMax, xMin, Helpers.colorToScalar(settings.crosshairColor), 2); - Imgproc.line(input.getLeft(), yMax, yMin, Helpers.colorToScalar(settings.crosshairColor), 2); - } - // processBuffer.copyTo(outputMat); // processBuffer.release(); } else { // input.getLeft().copyTo(outputMat); } - long processTime = System.nanoTime() - processStartNanos; return Pair.of(input.getLeft(), processTime); } public static class Draw2dContoursSettings { public boolean showCentroid = false; - public boolean showCrosshair = false; public boolean showMultiple = false; public int boxOutlineSize = 0; public boolean showRotatedBox = false; public boolean showMaximumBox = false; public Color centroidColor = Color.GREEN; - public Color crosshairColor = Color.GREEN; public Color rotatedBoxColor = Color.BLUE; public Color maximumBoxColor = Color.RED; } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dCrosshairPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dCrosshairPipe.java new file mode 100644 index 000000000..3f215ea38 --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/Draw2dCrosshairPipe.java @@ -0,0 +1,84 @@ +package com.chameleonvision.vision.pipeline.pipes; + +import com.chameleonvision.util.Helpers; +import com.chameleonvision.vision.enums.CalibrationMode; +import com.chameleonvision.vision.pipeline.Pipe; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; +import org.opencv.core.Point; +import org.opencv.imgproc.Imgproc; + +import java.awt.*; +import java.util.List; + +public class Draw2dCrosshairPipe implements Pipe>, Mat> { + + //Settings + private Draw2dCrosshairPipeSettings crosshairSettings; + private CalibrationMode calibrationMode; + private List calibrationPoint; + private double calibrationM, calibrationB; + + + private Point xMax = new Point(), xMin = new Point(), yMax = new Point(), yMin = new Point(); + + public Draw2dCrosshairPipe(Draw2dCrosshairPipeSettings crosshairSettings, CalibrationMode calibrationMode, List calibrationPoint, double calibrationM, double calibrationB) { + setConfig(crosshairSettings, calibrationMode, calibrationPoint, calibrationM, calibrationB); + } + + public void setConfig(Draw2dCrosshairPipeSettings crosshairSettings, CalibrationMode calibrationMode, List calibrationPoint, double calibrationM, double calibrationB) { + this.crosshairSettings = crosshairSettings; + this.calibrationMode = calibrationMode; + this.calibrationPoint = calibrationPoint; + this.calibrationM = calibrationM; + this.calibrationB = calibrationB; + } + + @Override + public Pair run(Pair> inputPair) { + long processStartNanos = System.nanoTime(); + Mat image = inputPair.getLeft(); + List targets = inputPair.getRight(); + double x = 0, y = 0, scale = image.cols() / 32.0; + + drawCrosshair: + if (this.crosshairSettings.showCrosshair) { + x = image.cols() / 2; + y = image.rows() / 2; + switch (this.calibrationMode) { + case Single: + if(this.calibrationPoint.isEmpty()) + { + this.calibrationPoint.add(x); + this.calibrationPoint.add(y); + } + x = this.calibrationPoint.get(0).intValue(); + y = this.calibrationPoint.get(1).intValue(); + break; + case Dual: +// if (targets != null && !targets.isEmpty()) { +// x = targets.get(0).calibratedX; +// y = targets.get(0).calibratedY; +// //TODO dual point calibration crosshair checks +// } else +// break drawCrosshair; + break; + } + xMax.set(new double[]{x + scale, y}); + xMin.set(new double[]{x - scale, y}); + yMax.set(new double[]{x, y + scale}); + yMin.set(new double[]{x, y - scale}); + Imgproc.line(inputPair.getLeft(), xMax, xMin, Helpers.colorToScalar(this.crosshairSettings.crosshairColor), 2); + Imgproc.line(inputPair.getLeft(), yMax, yMin, Helpers.colorToScalar(this.crosshairSettings.crosshairColor), 2); + } + + long processTime = System.nanoTime() - processStartNanos; + return Pair.of(inputPair.getLeft(), processTime); + } + + public static class Draw2dCrosshairPipeSettings { + public boolean showCrosshair = true; + public Color crosshairColor = Color.GREEN; + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/DrawSolvePNPPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/DrawSolvePNPPipe.java new file mode 100644 index 000000000..3eaad04fb --- /dev/null +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/DrawSolvePNPPipe.java @@ -0,0 +1,109 @@ +package com.chameleonvision.vision.pipeline.pipes; + +import com.chameleonvision.config.CameraCalibrationConfig; +import com.chameleonvision.util.Helpers; +import com.chameleonvision.vision.pipeline.Pipe; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; +import org.apache.commons.lang3.tuple.Pair; +import org.opencv.calib3d.Calib3d; +import org.opencv.core.*; +import org.opencv.core.Point; +import org.opencv.imgproc.Imgproc; + +import java.awt.*; +import java.util.List; + +public class DrawSolvePNPPipe implements Pipe>, Mat> { + + private MatOfPoint3f boxCornerMat = new MatOfPoint3f(); + + public Scalar color = Helpers.colorToScalar(Color.GREEN); + + public DrawSolvePNPPipe(CameraCalibrationConfig settings) { + setConfig(settings); + setObjectBox(14.5, 6, 2); + } + + public void setObjectBox(double targetWidth, double targetHeight, double targetDepth) { + // implementation from 5190 Green Hope Falcons + + boxCornerMat.release(); + boxCornerMat = new MatOfPoint3f( + new Point3(-targetWidth/2d, -targetHeight/2d, 0), + new Point3(-targetWidth/2d, targetHeight/2d, 0), + new Point3(targetWidth/2d, targetHeight/2d, 0), + new Point3(targetWidth/2d, -targetHeight/2d, 0), + new Point3(-targetWidth/2d, -targetHeight/2d, -targetDepth), + new Point3(-targetWidth/2d, targetHeight/2d, -targetDepth), + new Point3(targetWidth/2d, targetHeight/2d, -targetDepth), + new Point3(targetWidth/2d, -targetHeight/2d, -targetDepth) + ); + } + + private Mat cameraMatrix = new Mat(); + private MatOfDouble distortionCoefficients = new MatOfDouble(); + + public void setConfig(CameraCalibrationConfig config) { + if(config == null) { + System.err.println("got passed a null config! Returning..."); + return; + } + setConfig(config.getCameraMatrixAsMat(), config.getDistortionCoeffsAsMat()); + } + + public void setConfig(Mat cameraMatrix_, MatOfDouble distortionMatrix_) { + this.cameraMatrix = cameraMatrix_; + this.distortionCoefficients = distortionMatrix_; + } + + @Override + public Pair run(Pair> targets) { + long processStartNanos = System.nanoTime(); + + var image = targets.getLeft(); + for(var it : targets.getRight()) { + MatOfPoint2f imagePoints = new MatOfPoint2f(); + try { + Calib3d.projectPoints(boxCornerMat, it.rVector, it.tVector, this.cameraMatrix, this.distortionCoefficients, imagePoints, new Mat() , 0); + } catch (Exception e) { + e.printStackTrace(); + } + var pts = imagePoints.toList(); + + // draw left and right targets if possible + if(it.leftRightDualTargetPair != null) { + var left = it.leftRightDualTargetPair.getLeft(); + var right = it.leftRightDualTargetPair.getRight(); + Imgproc.rectangle(image, left.tl(), left.br(), new Scalar(200, 200, 0), 4); + Imgproc.rectangle(image, right.tl(), right.br(), new Scalar(200, 200, 0), 2); + } + + // draw corners + for(int i = 0; i < it.imageCornerPoints.rows(); i++) { + var point = new Point(it.imageCornerPoints.get(i, 0)); + Imgproc.circle(image, point, 4, new Scalar(0, 255, 0), 5); + } + + // sketch out floor + Imgproc.line(image, pts.get(0), pts.get(1), new Scalar(0, 255, 0), 3); + Imgproc.line(image, pts.get(1), pts.get(2), new Scalar(0, 255, 0), 3); + Imgproc.line(image, pts.get(2), pts.get(3), new Scalar(0, 255, 0), 3); + Imgproc.line(image, pts.get(3), pts.get(0), new Scalar(0, 255, 0), 3); + + // draw pillars + Imgproc.line(image, pts.get(0), pts.get(4), new Scalar(255, 0, 0), 3); + Imgproc.line(image, pts.get(1), pts.get(5), new Scalar(255, 0, 0), 3); + Imgproc.line(image, pts.get(2), pts.get(6), new Scalar(255, 0, 0), 3); + Imgproc.line(image, pts.get(3), pts.get(7), new Scalar(255, 0, 0), 3); + + // draw top + Imgproc.line(image, pts.get(4), pts.get(5), new Scalar(0, 0, 255), 3); + Imgproc.line(image, pts.get(5), pts.get(6), new Scalar(0, 0, 255), 3); + Imgproc.line(image, pts.get(6), pts.get(7), new Scalar(0, 0, 255), 3); + Imgproc.line(image, pts.get(7), pts.get(4), new Scalar(0, 0, 255), 3); + } + + long processTime = System.nanoTime() - processStartNanos; + return Pair.of(image, processTime); + } +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java index 5b8a67319..2d5034118 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/ErodeDilatePipe.java @@ -1,5 +1,6 @@ package com.chameleonvision.vision.pipeline.pipes; +import com.chameleonvision.vision.pipeline.Pipe; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; import org.opencv.core.Size; @@ -11,9 +12,6 @@ public class ErodeDilatePipe implements Pipe { private boolean dilate; private Mat kernel; - private Mat processBuffer = new Mat(); - private Mat outputMat = new Mat(); - public ErodeDilatePipe(boolean erode, boolean dilate, int kernelSize) { this.erode = erode; this.dilate = dilate; diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java index 121216214..282c711db 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FilterContoursPipe.java @@ -2,6 +2,7 @@ package com.chameleonvision.vision.pipeline.pipes; import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.util.MathHandler; +import com.chameleonvision.vision.pipeline.Pipe; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.MatOfPoint; import org.opencv.core.MatOfPoint2f; diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java index bf57d57d1..e2426dbb5 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/FindContoursPipe.java @@ -1,5 +1,6 @@ package com.chameleonvision.vision.pipeline.pipes; +import com.chameleonvision.vision.pipeline.Pipe; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.Mat; import org.opencv.core.MatOfPoint; diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java index 93b64f25b..42cfd6d96 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/GroupContoursPipe.java @@ -3,6 +3,8 @@ package com.chameleonvision.vision.pipeline.pipes; import com.chameleonvision.util.MathHandler; import com.chameleonvision.vision.enums.TargetGroup; import com.chameleonvision.vision.enums.TargetIntersection; +import com.chameleonvision.vision.pipeline.Pipe; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; import org.apache.commons.lang3.tuple.Pair; import org.opencv.core.*; import org.opencv.imgproc.Imgproc; @@ -13,7 +15,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -public class GroupContoursPipe implements Pipe, List> { +public class GroupContoursPipe implements Pipe, List> { private static final Comparator sortByMomentsX = Comparator.comparingDouble(GroupContoursPipe::calcMomentsX); @@ -21,7 +23,7 @@ public class GroupContoursPipe implements Pipe, List groupedContours = new ArrayList<>(); + private List groupedContours = new ArrayList<>(); private MatOfPoint2f intersectMatA = new MatOfPoint2f(); private MatOfPoint2f intersectMatB = new MatOfPoint2f(); @@ -36,9 +38,10 @@ public class GroupContoursPipe implements Pipe, List, Long> run(List input) { + public Pair, Long> run(List input) { long processStartNanos = System.nanoTime(); + groupedContours.forEach(StandardCVPipeline.TrackedTarget::release); groupedContours.clear(); if (input.size() > (group.equals(TargetGroup.Single) ? 0 : 1)) { @@ -55,7 +58,9 @@ public class GroupContoursPipe implements Pipe, List, List, List, List> { + + private Double tilt_angle; + private MatOfPoint3f objPointsMat = new MatOfPoint3f(); + private Mat rVec = new Mat(); + private Mat tVec = new Mat(); + private Mat rodriguez = new Mat(); + private Mat pzero_world = new Mat(); + private Mat cameraMatrix = new Mat(); + Mat rot_inv = new Mat(); + Mat kMat = new Mat(); + private MatOfDouble distortionCoefficients = new MatOfDouble(); + private List poseList = new ArrayList<>(); + Comparator leftRightComparator = Comparator.comparingDouble(point -> point.x); + Comparator verticalComparator = Comparator.comparingDouble(point -> point.y); + private double distanceDivisor = 1.0; + Mat scaledTvec = new Mat(); + + public SolvePNPPipe(StandardCVPipelineSettings settings, CameraCalibrationConfig calibration, Rotation2d tilt) { + super(); + setCameraCoeffs(calibration); + setTarget(settings.targetWidth, settings.targetHeight); + this.tilt_angle = tilt.getRadians(); + } + + public void setTarget(double targetWidth, double targetHeight) { + // order is left top, left bottom, right bottom, right top + + List corners = List.of( + new Point3(-targetWidth / 2.0, targetHeight / 2.0, 0.0), + new Point3(-targetWidth / 2.0, -targetHeight / 2.0, 0.0), + new Point3(targetWidth / 2.0, -targetHeight / 2.0, 0.0), + new Point3(targetWidth / 2.0, targetHeight / 2.0, 0.0) + ); + setObjectCorners(corners); + } + + public void setObjectCorners(List objectCorners) { + objPointsMat.release(); + objPointsMat = new MatOfPoint3f(); + objPointsMat.fromList(objectCorners); + } + + public void setConfig(StandardCVPipelineSettings settings, CameraCalibrationConfig camConfig, Rotation2d tilt) { + setCameraCoeffs(camConfig); + setTarget(settings.targetWidth, settings.targetHeight); + tilt_angle = tilt.getRadians(); + } + + private void setCameraCoeffs(CameraCalibrationConfig settings) { + if(settings == null) { + System.err.println("SolvePNP can only run on a calibrated resolution, and this one is not! Please calibrate to use solvePNP."); + return; + } + if(cameraMatrix != settings.getCameraMatrixAsMat()) { + cameraMatrix.release(); + settings.getCameraMatrixAsMat().copyTo(cameraMatrix); + } + if(distortionCoefficients != settings.getDistortionCoeffsAsMat()) { + distortionCoefficients.release(); + settings.getDistortionCoeffsAsMat().copyTo(distortionCoefficients); + } + this.distanceDivisor = settings.squareSize; + } + + @Override + public Pair, Long> run(List targets) { + long processStartNanos = System.nanoTime(); + poseList.clear(); + for(var target: targets) { + var corners = (target.leftRightDualTargetPair != null) ? findCorner2019(target) : findBoundingBoxCorners(target); + var pose = calculatePose(corners, target); + if(pose != null) poseList.add(pose); + } + long processTime = System.nanoTime() - processStartNanos; + return Pair.of(poseList, processTime); + } + + private MatOfPoint2f findCorner2019(StandardCVPipeline.TrackedTarget target) { + if(target.leftRightDualTargetPair == null) return null; + + var left = target.leftRightDualTargetPair.getLeft(); + var right = target.leftRightDualTargetPair.getRight(); + + // flip if the "left" target is to the right + if(left.x > right.x) { + var temp = left; + left = right; + right = temp; + } + + var points = new MatOfPoint2f(); + points.fromArray( + new Point(left.x, left.y + left.height), + new Point(left.x, left.y), + new Point(right.x + right.width, right.y), + new Point(right.x + right.width, right.y + right.height) + ); + return points; + } + + private MatOfPoint2f findCornerMinAreaRect(StandardCVPipeline.TrackedTarget target) { + if(target.leftRightRotatedRect == null) return null; + + var centroid = target.minAreaRect.center; + + var left = target.leftRightRotatedRect.getLeft(); + var right = target.leftRightRotatedRect.getRight(); + + // flip if the "left" target is to the right + if(left.center.x > right.center.x) { + var temp = left; + left = right; + right = temp; + } + + var leftPoints = new Point[4]; + left.points(leftPoints); + var rightPoints = new Point[4]; + right.points(rightPoints); + ArrayList combinedList = new ArrayList<>(List.of(leftPoints)); + combinedList.addAll(List.of(rightPoints)); + + // start looking in the top left quadrant + Comparator distanceProvider = Comparator.comparingDouble((Point point) -> FastMath.sqrt(FastMath.pow(centroid.x - point.x, 2) + FastMath.pow(centroid.y - point.y, 2))); + + var tl = combinedList.stream().filter(point -> point.x < centroid.x && point.y < centroid.y).max(distanceProvider).get(); + var tr = combinedList.stream().filter(point -> point.x > centroid.x && point.y < centroid.y).max(distanceProvider).get(); + var bl = combinedList.stream().filter(point -> point.x < centroid.x && point.y > centroid.y).max(distanceProvider).get(); + var br = combinedList.stream().filter(point -> point.x > centroid.x && point.y > centroid.y).max(distanceProvider).get(); + + boundingBoxResultMat.release(); + boundingBoxResultMat.fromList(List.of(tl, bl, br, tr)); + + return boundingBoxResultMat; + } + + private MatOfPoint2f findBoundingBoxCorners(StandardCVPipeline.TrackedTarget target) { + +// List> list = new ArrayList<>(); +// // find the corners based on the bounding box +// // order is left top, left bottom, right bottom, right top + + // extract the corners + var points = new Point[4]; + target.minAreaRect.points(points); + + // find the tl/tr/bl/br corners + // first, min by left/right + var list_ = Arrays.asList(points); + list_.sort(leftRightComparator); + // of this, we now have left and right + // sort to get top and bottom + var left = new ArrayList<>(List.of(list_.get(0), list_.get(1))); + left.sort(verticalComparator); + var right = new ArrayList<>(List.of(list_.get(2), list_.get(3))); + right.sort(verticalComparator); + + // tl tr bl br + var tl = left.get(0); + var bl = left.get(1); + var tr = right.get(0); + var br = right.get(1); + + boundingBoxResultMat.release(); + boundingBoxResultMat.fromList(List.of(tl, bl, br, tr)); + + return boundingBoxResultMat; + } + + MatOfPoint2f boundingBoxResultMat = new MatOfPoint2f(); + MatOfPoint2f goodFeaturesResultMat = new MatOfPoint2f(); + + private Mat dstNorm = new Mat(); + private Mat dstNormScaled = new Mat(); + List tempCornerList = new ArrayList<>(); + + /** + * Find the corners in an image. + * @param targetImage the image to find corners in. + * @return the corners found in the image. + */ + @Deprecated + private List findCornerHarris(Mat targetImage) { + + // convert the image to greyscale + var gray = new Mat(); + Imgproc.cvtColor(targetImage, gray, Imgproc.COLOR_BGR2GRAY); + Mat dst = Mat.zeros(targetImage.size(), CvType.CV_8U); + + // constants + final int blockSize = 2; + final int apertureSize = 3; + final double k = 0.04; + final int threshold = 200; + + /// Detecting corners + Imgproc.cornerHarris(gray, dst, blockSize, apertureSize, k); + + /// Normalizing + Core.normalize(dst, dstNorm, 0, 255, Core.NORM_MINMAX); + Core.convertScaleAbs(dstNorm, dstNormScaled); + + /// Drawing a circle around corners + float[] dstNormData = new float[(int) (dstNorm.total() * dstNorm.channels())]; + dstNorm.get(0, 0, dstNormData); + + tempCornerList.clear(); + for (int i = 0; i < dstNorm.rows(); i++) { + for (int j = 0; j < dstNorm.cols(); j++) { + if ((int) dstNormData[i * dstNorm.cols() + j] > threshold) { + tempCornerList.add(new Point(j, i)); + } + } + } + + return tempCornerList; + } + + @Deprecated + private MatOfPoint2f findGoodFeaturesToTrack2019(StandardCVPipeline.TrackedTarget target, Mat srcImage) { + +// Imgproc.approxPolyDP(new MatOfPoint2f(max_contour.toArray()),approx,epsilon,true); + + // start by looking at the targets + var leftRight = target.leftRightDualTargetPair; + var reverse = (leftRight.getLeft().x < leftRight.getRight().x); + var left = reverse ? leftRight.getLeft() : leftRight.getRight(); + var right = !reverse ? leftRight.getLeft() : leftRight.getRight(); + var boundingTl = left.tl(); + var boundingBr = right.br(); + + var slightlyBiggerTl = new Point( + Math.max(0, boundingTl.x - 5), + Math.max(0, boundingTl.y - 5) + ); + + var slightlyBiggerBr = new Point( + Math.min(srcImage.rows(), boundingBr.x + 5), + Math.min(srcImage.cols(), boundingBr.y + 5) + ); + var rect = new Rect(slightlyBiggerTl, slightlyBiggerBr); + + var croppedImage = srcImage.submat(rect); + var corners = new MatOfPoint(); + Imgproc.goodFeaturesToTrack(croppedImage, corners, 8,0.5,5); + + List cornerList = new ArrayList<>(corners.toList()); + if(cornerList.size() != 8 && cornerList.size() != 4) return null; + cornerList.sort(leftRightComparator); + + cornerList = cornerList.stream().map(point -> + new Point(point.x + slightlyBiggerTl.x, point.y + slightlyBiggerTl.y)) + .collect(Collectors.toList()); + + // of these, we want the two leftmost and two rightmost points + var left1 = cornerList.get(0); + var left2 = cornerList.get(1); + var right1 = cornerList.get(0); + var right2 = cornerList.get(1); + + // TODO maximize distance from the center rather than naively assume the leftmost and rightmost + // will have to do per quadrant + + var leftOrder = left1.y < left2.y; + var rightOrder = right1.y < right2.y; + + var tl = leftOrder ? left1 : left2; + var bl = !leftOrder ? left1 : left2; + var tr = rightOrder ? right1 : right2; + var br = !rightOrder ? right1 : right2; + + goodFeaturesResultMat.release(); + goodFeaturesResultMat.fromList(List.of(tl, bl, br, tr)); + + return goodFeaturesResultMat; + } + + private StandardCVPipeline.TrackedTarget calculatePose(MatOfPoint2f imageCornerPoints, StandardCVPipeline.TrackedTarget target) { + if(objPointsMat.rows() != imageCornerPoints.rows() || cameraMatrix.rows() < 2 || distortionCoefficients.cols() < 4) { + System.err.println("can't do solvePNP with invalid params!"); + return null; + } + + imageCornerPoints.copyTo(target.imageCornerPoints); + + try { + Calib3d.solvePnP(objPointsMat, imageCornerPoints, cameraMatrix, distortionCoefficients, rVec, tVec); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + + // Algorithm from team 5190 Green Hope Falcons + +// var tilt_angle = 0.0; // TODO add to settings + + var x = tVec.get(0, 0)[0]; + var z = FastMath.sin(tilt_angle) * tVec.get(1, 0)[0] + tVec.get(2, 0)[0] * FastMath.cos(tilt_angle); + + // distance in the horizontal plane between camera and target + var distance = FastMath.sqrt(x * x + z * z); + + // horizontal angle between center camera and target + @SuppressWarnings("SuspiciousNameCombination") + var angle1 = FastMath.atan2(x, z); + + Calib3d.Rodrigues(rVec, rodriguez); + Core.transpose(rodriguez, rot_inv); + + // This should be pzero_world = numpy.matmul(rot_inv, -tvec) +// pzero_world = rot_inv.mul(matScale(tVec, -1)); + scaledTvec = matScale(tVec, -1); + Core.gemm(rot_inv, scaledTvec, 1, kMat, 0, pzero_world); + + var angle2 = FastMath.atan2(pzero_world.get(0, 0)[0], pzero_world.get(2, 0)[0]); + + var targetAngle = -angle1; // radians + var targetRotation = -angle2; // radians + var targetDistance = distance * 25.4 / 1000d / distanceDivisor; // This should be meters + + var targetLocation = new Translation2d(targetDistance * FastMath.cos(targetAngle), targetDistance * FastMath.sin(targetAngle)); + target.cameraRelativePose = new Pose2d(targetLocation, new Rotation2d(targetRotation)); + target.rVector = rVec; + target.tVector = tVec; + + return target; + } + + /** + * Element-wise scale a matrix by a given factor + * @param src the source matrix + * @param factor by how much to scale each element + * @return the scaled matrix + */ + public Mat matScale(Mat src, double factor) { + Mat dst = new Mat(src.rows(),src.cols(),src.type()); + Scalar s = new Scalar(factor); // TODO check if we need to add more elements to this + Core.multiply(src, s, dst); + return dst; + } + +} diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java index ee10cf099..4435a8874 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SortContoursPipe.java @@ -2,32 +2,34 @@ package com.chameleonvision.vision.pipeline.pipes; import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.vision.enums.SortMode; +import com.chameleonvision.vision.pipeline.Pipe; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.math3.util.FastMath; -import org.opencv.core.RotatedRect; +import org.opencv.core.*; import java.util.ArrayList; import java.util.Comparator; import java.util.List; -public class SortContoursPipe implements Pipe, List> { +public class SortContoursPipe implements Pipe, List> { - private final Comparator SortByCentermostComparator = Comparator.comparingDouble(this::calcCenterDistance); + private final Comparator SortByCentermostComparator = Comparator.comparingDouble(this::calcSquareCenterDistance); - private static final Comparator SortByLargestComparator = (rect1, rect2) -> Double.compare(rect2.size.area(), rect1.size.area()); - private static final Comparator SortBySmallestComparator = SortByLargestComparator.reversed(); + private static final Comparator SortByLargestComparator = (rect1, rect2) -> Double.compare(rect2.minAreaRect.size.area(), rect1.minAreaRect.size.area()); + private static final Comparator SortBySmallestComparator = SortByLargestComparator.reversed(); - private static final Comparator SortByHighestComparator = (rect1, rect2) -> Double.compare(rect2.center.y, rect1.center.y); - private static final Comparator SortByLowestComparator = SortByHighestComparator.reversed(); + private static final Comparator SortByHighestComparator = (rect1, rect2) -> Double.compare(rect1.minAreaRect.center.y, rect2.minAreaRect.center.y); + private static final Comparator SortByLowestComparator = SortByHighestComparator.reversed(); - private static final Comparator SortByLeftmostComparator = Comparator.comparingDouble(rect -> rect.center.x); - private static final Comparator SortByRightmostComparator = SortByLeftmostComparator.reversed(); + public static final Comparator SortByLeftmostComparator = Comparator.comparingDouble(target -> target.minAreaRect.center.x); + private static final Comparator SortByRightmostComparator = SortByLeftmostComparator.reversed(); private SortMode sort; private CaptureStaticProperties camProps; private int maxTargets; - private List sortedContours = new ArrayList<>(); + private List sortedContours = new ArrayList<>(); public SortContoursPipe(SortMode sort, CaptureStaticProperties camProps, int maxTargets) { this.sort = sort; @@ -42,13 +44,13 @@ public class SortContoursPipe implements Pipe, List, Long> run(List input) { + public Pair, Long> run(List input) { long processStartNanos = System.nanoTime(); sortedContours.clear(); if (input.size() > 0) { - sortedContours.addAll(input.subList(0, Math.min(input.size(), maxTargets - 1))); + sortedContours.addAll(input); switch (sort) { case Largest: @@ -77,11 +79,16 @@ public class SortContoursPipe implements Pipe, List(sortedContours.subList(0, Math.min(input.size(), maxTargets - 1))); + sortedContours.subList(Math.min(input.size(), maxTargets - 1), sortedContours.size()).forEach(StandardCVPipeline.TrackedTarget::release); + sortedContours.clear(); + sortedContours = new ArrayList<>(); + long processTime = System.nanoTime() - processStartNanos; - return Pair.of(sortedContours, processTime); + return Pair.of(sublistedContors, processTime); } - private double calcCenterDistance(RotatedRect rect) { - return FastMath.sqrt(FastMath.pow(camProps.centerX - rect.center.x, 2) + FastMath.pow(camProps.centerY - rect.center.y, 2)); + private double calcSquareCenterDistance(StandardCVPipeline.TrackedTarget rect) { + return FastMath.sqrt(FastMath.pow(camProps.centerX - rect.minAreaRect.center.x, 2) + FastMath.pow(camProps.centerY - rect.minAreaRect.center.y, 2)); } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java index 058e955d6..063bcdbb0 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java +++ b/chameleon-server/src/main/java/com/chameleonvision/vision/pipeline/pipes/SpeckleRejectPipe.java @@ -1,7 +1,10 @@ package com.chameleonvision.vision.pipeline.pipes; +import com.chameleonvision.vision.pipeline.Pipe; import org.apache.commons.lang3.tuple.Pair; +import org.opencv.core.Mat; import org.opencv.core.MatOfPoint; +import org.opencv.core.MatOfPoint2f; import org.opencv.imgproc.Imgproc; import java.util.ArrayList; @@ -25,7 +28,9 @@ public class SpeckleRejectPipe implements Pipe, List, Long> run(List input) { long processStartNanos = System.nanoTime(); + despeckledContours.forEach(MatOfPoint::release); despeckledContours.clear(); + despeckledContours = new ArrayList<>(); if (input.size() > 0) { double averageArea = 0.0; diff --git a/chameleon-server/src/main/java/com/chameleonvision/web/RequestHandler.java b/chameleon-server/src/main/java/com/chameleonvision/web/RequestHandler.java index 34bff6030..d7c2a102c 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/web/RequestHandler.java +++ b/chameleon-server/src/main/java/com/chameleonvision/web/RequestHandler.java @@ -1,21 +1,35 @@ package com.chameleonvision.web; +import com.chameleonvision.Exceptions.DuplicatedKeyException; import com.chameleonvision.config.ConfigManager; import com.chameleonvision.network.NetworkIPMode; import com.chameleonvision.vision.VisionManager; import com.chameleonvision.vision.VisionProcess; -import com.chameleonvision.vision.camera.CameraCapture; import com.chameleonvision.vision.camera.USBCameraCapture; +import com.chameleonvision.vision.pipeline.CVPipelineSettings; +import com.chameleonvision.vision.pipeline.PipelineManager; +import com.chameleonvision.vision.pipeline.impl.Calibrate3dPipeline; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipelineSettings; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import edu.wpi.cscore.VideoMode; +import edu.wpi.first.wpilibj.geometry.Rotation2d; import io.javalin.http.Context; +import io.javalin.http.Handler; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; import java.util.Map; public class RequestHandler { + private static final ObjectMapper kObjectMapper = new ObjectMapper(); + public static void onGeneralSettings(Context ctx) { - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = kObjectMapper; try { Map map = objectMapper.readValue(ctx.body(), Map.class); @@ -35,21 +49,67 @@ public class RequestHandler { } } + public static void onDuplicatePipeline(Context ctx) { + ObjectMapper objectMapper = kObjectMapper; + try { + Map data = objectMapper.readValue(ctx.body(), Map.class); + + int cameraIndex = (Integer) data.getOrDefault("camera", -1); + + var pipelineIndex = (Integer) data.get("pipeline"); + StandardCVPipelineSettings origPipeline = (StandardCVPipelineSettings) VisionManager.getCurrentUIVisionProcess().pipelineManager.getPipeline(pipelineIndex).settings; + String tmp = objectMapper.writeValueAsString(origPipeline); + StandardCVPipelineSettings newPipeline = objectMapper.readValue(tmp, StandardCVPipelineSettings.class); + + if (cameraIndex == -1) { // same camera + + VisionManager.getCurrentUIVisionProcess().pipelineManager.duplicatePipeline(newPipeline); + + } else { // another camera + var cam = VisionManager.getVisionProcessByIndex(cameraIndex); + if (cam != null) { + if (cam.getCamera().getProperties().videoModes.size() < newPipeline.videoModeIndex) { + newPipeline.videoModeIndex = cam.getCamera().getProperties().videoModes.size() - 1; + } + if (newPipeline.is3D){ + var calibration = cam.getCamera().getCalibration(cam.getCamera().getProperties().getVideoMode(newPipeline.videoModeIndex)); + if (calibration == null){ + newPipeline.is3D = false; + } + } + VisionManager.getCurrentUIVisionProcess().pipelineManager.duplicatePipeline(newPipeline, cam); + ctx.status(200); + } else { + ctx.status(500); + } + } + } catch (JsonProcessingException | DuplicatedKeyException ex) { + ctx.status(500); + } + } + + public static void onCameraSettings(Context ctx) { - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = kObjectMapper; try { Map camSettings = objectMapper.readValue(ctx.body(), Map.class); VisionProcess currentVisionProcess = VisionManager.getCurrentUIVisionProcess(); USBCameraCapture currentCamera = currentVisionProcess.getCamera(); - double newFOV; + double newFOV, tilt; try { newFOV = (Double) camSettings.get("fov"); } catch (Exception ignored) { newFOV = (Integer) camSettings.get("fov"); } + try { + tilt = (Double) camSettings.get("tilt"); + } catch (Exception ignored) { + tilt = (Integer) camSettings.get("tilt"); + } currentCamera.getProperties().setFOV(newFOV); + currentCamera.getProperties().setTilt(Rotation2d.fromDegrees(tilt)); VisionManager.saveCurrentCameraSettings(); SocketHandler.sendFullSettings(); ctx.status(200); @@ -58,4 +118,67 @@ public class RequestHandler { ctx.status(500); } } + + public static void onCalibrationStart(Context ctx) throws JsonProcessingException { + PipelineManager pipeManager = VisionManager.getCurrentUIVisionProcess().pipelineManager; + ObjectMapper objectMapper = kObjectMapper; + var data = objectMapper.readValue(ctx.body(), Map.class); + int resolutionIndex = (Integer) data.get("resolution"); + double squareSize; + try { + squareSize = (Double) data.get("squareSize"); + } catch (Exception e) { + squareSize = (Integer) data.get("squareSize"); + } + // convert from mm to meters + pipeManager.calib3dPipe.setSquareSize(squareSize); + VisionManager.getCurrentUIVisionProcess().pipelineManager.calib3dPipe.settings.videoModeIndex = resolutionIndex; + VisionManager.getCurrentUIVisionProcess().pipelineManager.setCalibrationMode(true); + VisionManager.getCurrentUIVisionProcess().getCamera().setVideoMode(resolutionIndex); + } + + public static void onSnapshot(Context ctx) { + Calibrate3dPipeline calPipe = VisionManager.getCurrentUIVisionProcess().pipelineManager.calib3dPipe; + + calPipe.takeSnapshot(); + + HashMap toSend = new HashMap<>(); + toSend.put("snapshotCount", calPipe.getSnapshotCount()); + toSend.put("hasEnough", calPipe.hasEnoughSnapshots()); + + ctx.json(toSend); + ctx.status(200); + } + + public static void onCalibrationEnding(Context ctx) throws JsonProcessingException { + PipelineManager pipeManager = VisionManager.getCurrentUIVisionProcess().pipelineManager; + + var data = kObjectMapper.readValue(ctx.body(), Map.class); + double squareSize; + try { + squareSize = (Double) data.get("squareSize"); + } catch (Exception e) { + squareSize = (Integer) data.get("squareSize"); + } + pipeManager.calib3dPipe.setSquareSize(squareSize); + + System.out.println("Finishing Cal"); + if (pipeManager.calib3dPipe.hasEnoughSnapshots()) { + if (pipeManager.calib3dPipe.tryCalibration()) { + ctx.status(200); + } else { + System.err.println("CALFAIL"); + ctx.status(500); + } + } + pipeManager.setCalibrationMode(false); + ctx.status(200); + } + + public static void onPnpModel(Context ctx) throws JsonProcessingException { + System.out.println(ctx.body()); + ObjectMapper objectMapper = kObjectMapper; + List points = objectMapper.readValue(ctx.body(), List.class); + System.out.println(points); + } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/web/Server.java b/chameleon-server/src/main/java/com/chameleonvision/web/Server.java index b37393b39..a9617edbb 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/web/Server.java +++ b/chameleon-server/src/main/java/com/chameleonvision/web/Server.java @@ -30,6 +30,11 @@ public class Server { }); app.post("/api/settings/general", RequestHandler::onGeneralSettings); app.post("/api/settings/camera", RequestHandler::onCameraSettings); + app.post("/api/vision/duplicate", RequestHandler::onDuplicatePipeline); + app.post("/api/settings/startCalibration", RequestHandler::onCalibrationStart); + app.post("/api/settings/snapshot", RequestHandler::onSnapshot); + app.post("/api/settings/endCalibration", RequestHandler::onCalibrationEnding); + app.post("/api/vision/pnpModel", RequestHandler::onPnpModel); app.start(port); } } diff --git a/chameleon-server/src/main/java/com/chameleonvision/web/SocketHandler.java b/chameleon-server/src/main/java/com/chameleonvision/web/SocketHandler.java index 1a1b504bd..d8b7b5302 100644 --- a/chameleon-server/src/main/java/com/chameleonvision/web/SocketHandler.java +++ b/chameleon-server/src/main/java/com/chameleonvision/web/SocketHandler.java @@ -1,16 +1,21 @@ package com.chameleonvision.web; +import com.chameleonvision.config.CameraCalibrationConfig; import com.chameleonvision.config.ConfigManager; import com.chameleonvision.vision.VisionManager; import com.chameleonvision.vision.VisionProcess; import com.chameleonvision.vision.camera.CameraCapture; +import com.chameleonvision.vision.camera.CaptureStaticProperties; import com.chameleonvision.vision.camera.USBCameraCapture; +import com.chameleonvision.vision.enums.ImageRotationMode; import com.chameleonvision.vision.enums.StreamDivisor; import com.chameleonvision.vision.pipeline.CVPipeline; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; import com.chameleonvision.vision.pipeline.CVPipelineSettings; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import edu.wpi.cscore.VideoMode; import io.javalin.websocket.WsBinaryMessageContext; import io.javalin.websocket.WsCloseContext; import io.javalin.websocket.WsConnectContext; @@ -20,10 +25,9 @@ import org.msgpack.jackson.dataformat.MessagePackFactory; import java.lang.reflect.Field; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.*; import java.util.List; -import java.util.Map; +import java.util.stream.Collectors; public class SocketHandler { @@ -31,6 +35,8 @@ public class SocketHandler { private static List users; private static ObjectMapper objectMapper; + private static final Object broadcastLock = new Object(); + SocketHandler() { users = new ArrayList<>(); objectMapper = new ObjectMapper(new MessagePackFactory()); @@ -54,7 +60,7 @@ public class SocketHandler { VisionProcess currentProcess = VisionManager.getCurrentUIVisionProcess(); CameraCapture currentCamera = currentProcess.getCamera(); CVPipeline currentPipeline = currentProcess.pipelineManager.getCurrentPipeline(); - +// System.out.println("entry.getKey()+entry.getValue()= " + entry.getKey() + entry.getValue()); switch (entry.getKey()) { case "driverMode": { HashMap data = (HashMap) entry.getValue(); @@ -77,39 +83,17 @@ public class SocketHandler { VisionManager.saveCurrentCameraPipelines(); break; } - case "duplicatePipeline": { - HashMap pipelineVals = (HashMap) entry.getValue(); - int pipelineIndex = (int) pipelineVals.get("pipeline"); - int cameraIndex = (int) pipelineVals.get("camera"); - ObjectMapper mapper = new ObjectMapper(); - CVPipelineSettings origPipeline = currentProcess.pipelineManager.getPipeline(pipelineIndex).settings; - String val = mapper.writeValueAsString(origPipeline); - CVPipelineSettings newPipeline = mapper.readValue(val, origPipeline.getClass()); - - if (cameraIndex != -1) { - VisionProcess newProcess = VisionManager.getVisionProcessByIndex(cameraIndex); - if (newProcess != null) { - currentProcess.pipelineManager.duplicatePipeline(newPipeline, newProcess); - } else { - System.err.println("Failed to get destination camera for pipeline duplication!"); - } - } else { - currentProcess.pipelineManager.duplicatePipeline(newPipeline); - } - - VisionManager.saveCurrentCameraPipelines(); + case "addNewPipeline": { +// HashMap data = (HashMap) entry.getValue(); + String pipeName = (String) entry.getValue(); + // TODO: add to UI selection for new 2d/3d + currentProcess.pipelineManager.addNewPipeline(pipeName); sendFullSettings(); + VisionManager.saveCurrentCameraPipelines(); break; } case "command": { switch ((String) entry.getValue()) { - case "addNewPipeline": - // TODO: add to UI selection for new 2d/3d - boolean is3d = false; - currentProcess.pipelineManager.addNewPipeline(is3d); - sendFullSettings(); - VisionManager.saveCurrentCameraPipelines(); - break; case "deleteCurrentPipeline": currentProcess.pipelineManager.deleteCurrentPipeline(); sendFullSettings(); @@ -129,20 +113,54 @@ public class SocketHandler { sendFullSettings(); break; } + case "is3D": { + VisionManager.getCurrentUIVisionProcess().setIs3d((Boolean) entry.getValue()); + break; + } case "currentPipeline": { currentProcess.pipelineManager.setCurrentPipeline((Integer) entry.getValue()); sendFullSettings(); break; } + case "isPNPCalibration": { + currentProcess.pipelineManager.setCalibrationMode((Boolean) entry.getValue()); + break; + } + case "takeCalibrationSnapshot": { + currentProcess.pipelineManager.calib3dPipe.takeSnapshot(); + } default: { - // only change settings when we aren't in driver mode - if(currentProcess.pipelineManager.getDriverMode()) { + switch (entry.getKey()) {//Pre field value set + case "rotationMode": {//Create new CaptureStaticProperties with new width and height, reset crosshair calib + ImageRotationMode oldRot = currentPipeline.settings.rotationMode; + ImageRotationMode newRot = ImageRotationMode.class.getEnumConstants()[(Integer) entry.getValue()]; + CaptureStaticProperties prop = currentCamera.getProperties().getStaticProperties(); + int width, height; + if (oldRot.isRotated() != newRot.isRotated()) { + width = prop.mode.height; + height = prop.mode.width; + //Creates new video mode with new width and height to create new CaptureStaticProperties and applies it + currentCamera.getProperties().setStaticProperties(new CaptureStaticProperties( + new VideoMode(prop.mode.pixelFormat, width, height, prop.mode.fps), prop.fov)); + } + prop = currentCamera.getProperties().getStaticProperties(); + currentProcess.cameraStreamer.recalculateDivision(); + if (currentPipeline instanceof StandardCVPipeline) + ((StandardCVPipeline) currentPipeline).settings.point = Arrays.asList(prop.mode.width / 2, prop.mode.height / 2);//Reset Crosshair in single point calib + break; + } + + } + + + if (currentProcess.pipelineManager.getDriverMode()) { setField(currentProcess.pipelineManager.driverModePipeline.settings, entry.getKey(), entry.getValue()); } else { setField(currentPipeline.settings, entry.getKey(), entry.getValue()); } + //Post field value set switch (entry.getKey()) { case "exposure": { currentCamera.setExposure((Integer) entry.getValue()); @@ -152,11 +170,14 @@ public class SocketHandler { currentCamera.setBrightness((Integer) entry.getValue()); break; } - case "videoModeIndex":{ + case "videoModeIndex": { + if (currentPipeline instanceof StandardCVPipeline) + ((StandardCVPipeline) currentPipeline).settings.point = new ArrayList<>();//This will reset the calibration currentCamera.setVideoMode((Integer) entry.getValue()); + currentProcess.cameraStreamer.recalculateDivision(); break; } - case "streamDivisor":{ + case "streamDivisor": { currentProcess.cameraStreamer.setDivisor(StreamDivisor.values()[(Integer) entry.getValue()], true); break; } @@ -186,18 +207,22 @@ public class SocketHandler { } private static void broadcastMessage(Object obj, WsContext userToSkip) { - if (users != null) - for (WsContext user : users) { - if (userToSkip != null && user.getSessionId().equals(userToSkip.getSessionId())) { - continue; - } - try { - ByteBuffer b = ByteBuffer.wrap(objectMapper.writeValueAsBytes(obj)); - user.send(b); - } catch (JsonProcessingException e) { - e.printStackTrace(); + synchronized (broadcastLock) { + if (users != null) { + var userList = users; + for (WsContext user : userList) { + if (userToSkip != null && user.getSessionId().equals(userToSkip.getSessionId())) { + continue; + } + try { + ByteBuffer b = ByteBuffer.wrap(objectMapper.writeValueAsBytes(obj)); + user.send(b); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } } } + } } public static void broadcastMessage(Object obj) { @@ -236,9 +261,16 @@ public class SocketHandler { HashMap tmp = new HashMap<>(); VisionProcess currentVisionProcess = VisionManager.getCurrentUIVisionProcess(); USBCameraCapture currentCamera = VisionManager.getCurrentUIVisionProcess().getCamera(); + tmp.put("fov", currentCamera.getProperties().getFOV()); tmp.put("streamDivisor", currentVisionProcess.cameraStreamer.getDivisor().ordinal()); tmp.put("resolution", currentVisionProcess.getCamera().getProperties().getCurrentVideoModeIndex()); + tmp.put("tilt", currentVisionProcess.getCamera().getProperties().getTilt().getDegrees()); + + List calibrations = currentCamera.getAllCalibrationData().stream() + .map(CameraCalibrationConfig.UICameraCalibrationConfig::new).collect(Collectors.toList()); + tmp.put("calibration", calibrations); + return tmp; } diff --git a/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Pose2d.java b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Pose2d.java new file mode 100644 index 000000000..92c5a0219 --- /dev/null +++ b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Pose2d.java @@ -0,0 +1,271 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) 2019 FIRST. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +package edu.wpi.first.wpilibj.geometry; + +import java.io.IOException; +import java.util.Objects; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +/** + * Represents a 2d pose containing translational and rotational elements. + */ +@JsonSerialize(using = Pose2d.PoseSerializer.class) +@JsonDeserialize(using = Pose2d.PoseDeserializer.class) +public class Pose2d { + private final Translation2d m_translation; + private final Rotation2d m_rotation; + + /** + * Constructs a pose at the origin facing toward the positive X axis. + * (Translation2d{0, 0} and Rotation{0}) + */ + public Pose2d() { + m_translation = new Translation2d(); + m_rotation = new Rotation2d(); + } + + /** + * Constructs a pose with the specified translation and rotation. + * + * @param translation The translational component of the pose. + * @param rotation The rotational component of the pose. + */ + public Pose2d(Translation2d translation, Rotation2d rotation) { + m_translation = translation; + m_rotation = rotation; + } + + /** + * Convenience constructors that takes in x and y values directly instead of + * having to construct a Translation2d. + * + * @param x The x component of the translational component of the pose. + * @param y The y component of the translational component of the pose. + * @param rotation The rotational component of the pose. + */ + @SuppressWarnings("ParameterName") + public Pose2d(double x, double y, Rotation2d rotation) { + m_translation = new Translation2d(x, y); + m_rotation = rotation; + } + + /** + * Transforms the pose by the given transformation and returns the new + * transformed pose. + * + *

The matrix multiplication is as follows + * [x_new] [cos, -sin, 0][transform.x] + * [y_new] += [sin, cos, 0][transform.y] + * [t_new] [0, 0, 1][transform.t] + * + * @param other The transform to transform the pose by. + * @return The transformed pose. + */ + public Pose2d plus(Transform2d other) { + return transformBy(other); + } + + /** + * Returns the Transform2d that maps the one pose to another. + * + * @param other The initial pose of the transformation. + * @return The transform that maps the other pose to the current pose. + */ + public Transform2d minus(Pose2d other) { + final var pose = this.relativeTo(other); + return new Transform2d(pose.getTranslation(), pose.getRotation()); + } + + /** + * Returns the translation component of the transformation. + * + * @return The translational component of the pose. + */ + public Translation2d getTranslation() { + return m_translation; + } + + /** + * Returns the rotational component of the transformation. + * + * @return The rotational component of the pose. + */ + public Rotation2d getRotation() { + return m_rotation; + } + + /** + * Transforms the pose by the given transformation and returns the new pose. + * See + operator for the matrix multiplication performed. + * + * @param other The transform to transform the pose by. + * @return The transformed pose. + */ + public Pose2d transformBy(Transform2d other) { + return new Pose2d(m_translation.plus(other.getTranslation().rotateBy(m_rotation)), + m_rotation.plus(other.getRotation())); + } + + /** + * Returns the other pose relative to the current pose. + * + *

This function can often be used for trajectory tracking or pose + * stabilization algorithms to get the error between the reference and the + * current pose. + * + * @param other The pose that is the origin of the new coordinate frame that + * the current pose will be converted into. + * @return The current pose relative to the new origin pose. + */ + public Pose2d relativeTo(Pose2d other) { + var transform = new Transform2d(other, this); + return new Pose2d(transform.getTranslation(), transform.getRotation()); + } + + /** + * Obtain a new Pose2d from a (constant curvature) velocity. + * + *

See + * Controls Engineering in the FIRST Robotics Competition + * section on nonlinear pose estimation for derivation. + * + *

The twist is a change in pose in the robot's coordinate frame since the + * previous pose update. When the user runs exp() on the previous known + * field-relative pose with the argument being the twist, the user will + * receive the new field-relative pose. + * + *

"Exp" represents the pose exponential, which is solving a differential + * equation moving the pose forward in time. + * + * @param twist The change in pose in the robot's coordinate frame since the + * previous pose update. For example, if a non-holonomic robot moves forward + * 0.01 meters and changes angle by 0.5 degrees since the previous pose update, + * the twist would be Twist2d{0.01, 0.0, toRadians(0.5)} + * @return The new pose of the robot. + */ + @SuppressWarnings("LocalVariableName") + public Pose2d exp(Twist2d twist) { + double dx = twist.dx; + double dy = twist.dy; + double dtheta = twist.dtheta; + + double sinTheta = Math.sin(dtheta); + double cosTheta = Math.cos(dtheta); + + double s; + double c; + if (Math.abs(dtheta) < 1E-9) { + s = 1.0 - 1.0 / 6.0 * dtheta * dtheta; + c = 0.5 * dtheta; + } else { + s = sinTheta / dtheta; + c = (1 - cosTheta) / dtheta; + } + var transform = new Transform2d(new Translation2d(dx * s - dy * c, dx * c + dy * s), + new Rotation2d(cosTheta, sinTheta)); + + return this.plus(transform); + } + + /** + * Returns a Twist2d that maps this pose to the end pose. If c is the output + * of a.Log(b), then a.Exp(c) would yield b. + * + * @param end The end pose for the transformation. + * @return The twist that maps this to end. + */ + public Twist2d log(Pose2d end) { + final var transform = end.relativeTo(this); + final var dtheta = transform.getRotation().getRadians(); + final var halfDtheta = dtheta / 2.0; + + final var cosMinusOne = transform.getRotation().getCos() - 1; + + double halfThetaByTanOfHalfDtheta; + if (Math.abs(cosMinusOne) < 1E-9) { + halfThetaByTanOfHalfDtheta = 1.0 - 1.0 / 12.0 * dtheta * dtheta; + } else { + halfThetaByTanOfHalfDtheta = -(halfDtheta * transform.getRotation().getSin()) / cosMinusOne; + } + + Translation2d translationPart = transform.getTranslation().rotateBy( + new Rotation2d(halfThetaByTanOfHalfDtheta, -halfDtheta) + ).times(Math.hypot(halfThetaByTanOfHalfDtheta, halfDtheta)); + + return new Twist2d(translationPart.getX(), translationPart.getY(), dtheta); + } + + @Override + public String toString() { + return String.format("Pose2d(%s, %s)", m_translation, m_rotation); + } + + /** + * Checks equality between this Pose2d and another object. + * + * @param obj The other object. + * @return Whether the two objects are equal or not. + */ + @Override + public boolean equals(Object obj) { + if (obj instanceof Pose2d) { + return ((Pose2d) obj).m_translation.equals(m_translation) + && ((Pose2d) obj).m_rotation.equals(m_rotation); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(m_translation, m_rotation); + } + + static class PoseSerializer extends StdSerializer { + PoseSerializer() { + super(Pose2d.class); + } + + @Override + public void serialize( + Pose2d value, JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException { + + jgen.writeStartObject(); + jgen.writeObjectField("translation", value.m_translation); + jgen.writeObjectField("rotation", value.m_rotation); + jgen.writeEndObject(); + } + } + + static class PoseDeserializer extends StdDeserializer { + PoseDeserializer() { + super(Pose2d.class); + } + + @Override + public Pose2d deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + + Translation2d translation = + jp.getCodec().treeToValue(node.get("translation"), Translation2d.class); + Rotation2d rotation = jp.getCodec().treeToValue(node.get("rotation"), Rotation2d.class); + return new Pose2d(translation, rotation); + } + } +} diff --git a/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Rotation2d.java b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Rotation2d.java new file mode 100644 index 000000000..40950056a --- /dev/null +++ b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Rotation2d.java @@ -0,0 +1,251 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) 2019 FIRST. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +package edu.wpi.first.wpilibj.geometry; + +import java.io.IOException; +import java.util.Objects; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +/** + * A rotation in a 2d coordinate frame represented a point on the unit circle + * (cosine and sine). + */ +@JsonSerialize(using = Rotation2d.RotationSerializer.class) +@JsonDeserialize(using = Rotation2d.RotationDeserializer.class) +public class Rotation2d { + private final double m_value; + private final double m_cos; + private final double m_sin; + + /** + * Constructs a Rotation2d with a default angle of 0 degrees. + */ + public Rotation2d() { + m_value = 0.0; + m_cos = 1.0; + m_sin = 0.0; + } + + /** + * Constructs a Rotation2d with the given radian value. + * The x and y don't have to be normalized. + * + * @param value The value of the angle in radians. + */ + public Rotation2d(double value) { + m_value = value; + m_cos = Math.cos(value); + m_sin = Math.sin(value); + } + + /** + * Constructs a Rotation2d with the given x and y (cosine and sine) + * components. + * + * @param x The x component or cosine of the rotation. + * @param y The y component or sine of the rotation. + */ + @SuppressWarnings("ParameterName") + public Rotation2d(double x, double y) { + double magnitude = Math.hypot(x, y); + if (magnitude > 1e-6) { + m_sin = y / magnitude; + m_cos = x / magnitude; + } else { + m_sin = 0.0; + m_cos = 1.0; + } + m_value = Math.atan2(m_sin, m_cos); + } + + /** + * Constructs and returns a Rotation2d with the given degree value. + * + * @param degrees The value of the angle in degrees. + * @return The rotation object with the desired angle value. + */ + public static Rotation2d fromDegrees(double degrees) { + return new Rotation2d(Math.toRadians(degrees)); + } + + /** + * Adds two rotations together, with the result being bounded between -pi and + * pi. + * + *

For example, Rotation2d.fromDegrees(30) + Rotation2d.fromDegrees(60) = + * Rotation2d{-pi/2} + * + * @param other The rotation to add. + * @return The sum of the two rotations. + */ + public Rotation2d plus(Rotation2d other) { + return rotateBy(other); + } + + /** + * Subtracts the new rotation from the current rotation and returns the new + * rotation. + * + *

For example, Rotation2d.fromDegrees(10) - Rotation2d.fromDegrees(100) = + * Rotation2d{-pi/2} + * + * @param other The rotation to subtract. + * @return The difference between the two rotations. + */ + public Rotation2d minus(Rotation2d other) { + return rotateBy(other.unaryMinus()); + } + + /** + * Takes the inverse of the current rotation. This is simply the negative of + * the current angular value. + * + * @return The inverse of the current rotation. + */ + public Rotation2d unaryMinus() { + return new Rotation2d(-m_value); + } + + /** + * Multiplies the current rotation by a scalar. + * + * @param scalar The scalar. + * @return The new scaled Rotation2d. + */ + public Rotation2d times(double scalar) { + return new Rotation2d(m_value * scalar); + } + + /** + * Adds the new rotation to the current rotation using a rotation matrix. + * + *

The matrix multiplication is as follows: + * [cos_new] [other.cos, -other.sin][cos] + * [sin_new] = [other.sin, other.cos][sin] + * value_new = atan2(cos_new, sin_new) + * + * @param other The rotation to rotate by. + * @return The new rotated Rotation2d. + */ + public Rotation2d rotateBy(Rotation2d other) { + return new Rotation2d( + m_cos * other.m_cos - m_sin * other.m_sin, + m_cos * other.m_sin + m_sin * other.m_cos + ); + } + + /* + * Returns the radian value of the rotation. + * + * @return The radian value of the rotation. + */ + public double getRadians() { + return m_value; + } + + /** + * Returns the degree value of the rotation. + * + * @return The degree value of the rotation. + */ + public double getDegrees() { + return Math.toDegrees(m_value); + } + + /** + * Returns the cosine of the rotation. + * + * @return The cosine of the rotation. + */ + public double getCos() { + return m_cos; + } + + /** + * Returns the sine of the rotation. + * + * @return The sine of the rotation. + */ + public double getSin() { + return m_sin; + } + + /** + * Returns the tangent of the rotation. + * + * @return The tangent of the rotation. + */ + public double getTan() { + return m_sin / m_cos; + } + + @Override + public String toString() { + return String.format("Rotation2d(Rads: %.2f, Deg: %.2f)", m_value, Math.toDegrees(m_value)); + } + + /** + * Checks equality between this Rotation2d and another object. + * + * @param obj The other object. + * @return Whether the two objects are equal or not. + */ + @Override + public boolean equals(Object obj) { + if (obj instanceof Rotation2d) { + return Math.abs(((Rotation2d) obj).m_value - m_value) < 1E-9; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(m_value); + } + + static class RotationSerializer extends StdSerializer { + RotationSerializer() { + super(Rotation2d.class); + } + + @Override + public void serialize( + Rotation2d value, JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException { + + jgen.writeStartObject(); + jgen.writeNumberField("radians", value.m_value); + jgen.writeEndObject(); + } + } + + static class RotationDeserializer extends StdDeserializer { + RotationDeserializer() { + super(Rotation2d.class); + } + + @Override + public Rotation2d deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + double radians = node.get("radians").numberValue().doubleValue(); + + return new Rotation2d(radians); + } + } +} \ No newline at end of file diff --git a/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Transform2d.java b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Transform2d.java new file mode 100644 index 000000000..15f47851e --- /dev/null +++ b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Transform2d.java @@ -0,0 +1,106 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) 2019 FIRST. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +package edu.wpi.first.wpilibj.geometry; + +import java.util.Objects; + +/** + * Represents a transformation for a Pose2d. + */ +public class Transform2d { + private final Translation2d m_translation; + private final Rotation2d m_rotation; + + /** + * Constructs the transform that maps the initial pose to the final pose. + * + * @param initial The initial pose for the transformation. + * @param last The final pose for the transformation. + */ + public Transform2d(Pose2d initial, Pose2d last) { + // We are rotating the difference between the translations + // using a clockwise rotation matrix. This transforms the global + // delta into a local delta (relative to the initial pose). + m_translation = last.getTranslation().minus(initial.getTranslation()) + .rotateBy(initial.getRotation().unaryMinus()); + + m_rotation = last.getRotation().minus(initial.getRotation()); + } + + /** + * Constructs a transform with the given translation and rotation components. + * + * @param translation Translational component of the transform. + * @param rotation Rotational component of the transform. + */ + public Transform2d(Translation2d translation, Rotation2d rotation) { + m_translation = translation; + m_rotation = rotation; + } + + /** + * Constructs the identity transform -- maps an initial pose to itself. + */ + public Transform2d() { + m_translation = new Translation2d(); + m_rotation = new Rotation2d(); + } + + /** + * Scales the transform by the scalar. + * + * @param scalar The scalar. + * @return The scaled Transform2d. + */ + public Transform2d times(double scalar) { + return new Transform2d(m_translation.times(scalar), m_rotation.times(scalar)); + } + + /** + * Returns the translation component of the transformation. + * + * @return The translational component of the transform. + */ + public Translation2d getTranslation() { + return m_translation; + } + + /** + * Returns the rotational component of the transformation. + * + * @return Reference to the rotational component of the transform. + */ + public Rotation2d getRotation() { + return m_rotation; + } + + @Override + public String toString() { + return String.format("Transform2d(%s, %s)", m_translation, m_rotation); + } + + /** + * Checks equality between this Transform2d and another object. + * + * @param obj The other object. + * @return Whether the two objects are equal or not. + */ + @Override + public boolean equals(Object obj) { + if (obj instanceof Transform2d) { + return ((Transform2d) obj).m_translation.equals(m_translation) + && ((Transform2d) obj).m_rotation.equals(m_rotation); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(m_translation, m_rotation); + } +} diff --git a/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Translation2d.java b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Translation2d.java new file mode 100644 index 000000000..9d3f55038 --- /dev/null +++ b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Translation2d.java @@ -0,0 +1,243 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) 2019 FIRST. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +package edu.wpi.first.wpilibj.geometry; + +import java.io.IOException; +import java.util.Objects; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +/** + * Represents a translation in 2d space. + * This object can be used to represent a point or a vector. + * + *

This assumes that you are using conventional mathematical axes. + * When the robot is placed on the origin, facing toward the X direction, + * moving forward increases the X, whereas moving to the left increases the Y. + */ +@JsonSerialize(using = Translation2d.TranslationSerializer.class) +@JsonDeserialize(using = Translation2d.TranslationDeserializer.class) +@SuppressWarnings({"ParameterName", "MemberName"}) +public class Translation2d { + private final double m_x; + private final double m_y; + + /** + * Constructs a Translation2d with X and Y components equal to zero. + */ + public Translation2d() { + this(0.0, 0.0); + } + + /** + * Constructs a Translation2d with the X and Y components equal to the + * provided values. + * + * @param x The x component of the translation. + * @param y The y component of the translation. + */ + public Translation2d(double x, double y) { + m_x = x; + m_y = y; + } + + /** + * Calculates the distance between two translations in 2d space. + * + *

This function uses the pythagorean theorem to calculate the distance. + * distance = sqrt((x2 - x1)^2 + (y2 - y1)^2) + * + * @param other The translation to compute the distance to. + * @return The distance between the two translations. + */ + public double getDistance(Translation2d other) { + return Math.hypot(other.m_x - m_x, other.m_y - m_y); + } + + /** + * Returns the X component of the translation. + * + * @return The x component of the translation. + */ + public double getX() { + return m_x; + } + + /** + * Returns the Y component of the translation. + * + * @return The y component of the translation. + */ + public double getY() { + return m_y; + } + + /** + * Returns the norm, or distance from the origin to the translation. + * + * @return The norm of the translation. + */ + public double getNorm() { + return Math.hypot(m_x, m_y); + } + + /** + * Applies a rotation to the translation in 2d space. + * + *

This multiplies the translation vector by a counterclockwise rotation + * matrix of the given angle. + * [x_new] [other.cos, -other.sin][x] + * [y_new] = [other.sin, other.cos][y] + * + *

For example, rotating a Translation2d of {2, 0} by 90 degrees will return a + * Translation2d of {0, 2}. + * + * @param other The rotation to rotate the translation by. + * @return The new rotated translation. + */ + public Translation2d rotateBy(Rotation2d other) { + return new Translation2d( + m_x * other.getCos() - m_y * other.getSin(), + m_x * other.getSin() + m_y * other.getCos() + ); + } + + /** + * Adds two translations in 2d space and returns the sum. This is similar to + * vector addition. + * + *

For example, Translation2d{1.0, 2.5} + Translation2d{2.0, 5.5} = + * Translation2d{3.0, 8.0} + * + * @param other The translation to add. + * @return The sum of the translations. + */ + public Translation2d plus(Translation2d other) { + return new Translation2d(m_x + other.m_x, m_y + other.m_y); + } + + /** + * Subtracts the other translation from the other translation and returns the + * difference. + * + *

For example, Translation2d{5.0, 4.0} - Translation2d{1.0, 2.0} = + * Translation2d{4.0, 2.0} + * + * @param other The translation to subtract. + * @return The difference between the two translations. + */ + public Translation2d minus(Translation2d other) { + return new Translation2d(m_x - other.m_x, m_y - other.m_y); + } + + /** + * Returns the inverse of the current translation. This is equivalent to + * rotating by 180 degrees, flipping the point over both axes, or simply + * negating both components of the translation. + * + * @return The inverse of the current translation. + */ + public Translation2d unaryMinus() { + return new Translation2d(-m_x, -m_y); + } + + /** + * Multiplies the translation by a scalar and returns the new translation. + * + *

For example, Translation2d{2.0, 2.5} * 2 = Translation2d{4.0, 5.0} + * + * @param scalar The scalar to multiply by. + * @return The scaled translation. + */ + public Translation2d times(double scalar) { + return new Translation2d(m_x * scalar, m_y * scalar); + } + + /** + * Divides the translation by a scalar and returns the new translation. + * + *

For example, Translation2d{2.0, 2.5} / 2 = Translation2d{1.0, 1.25} + * + * @param scalar The scalar to multiply by. + * @return The reference to the new mutated object. + */ + public Translation2d div(double scalar) { + return new Translation2d(m_x / scalar, m_y / scalar); + } + + @Override + public String toString() { + return String.format("Translation2d(X: %.2f, Y: %.2f)", m_x, m_y); + } + + public static Translation2d fromRotation2d(Rotation2d rotation) { + return new Translation2d(rotation.getCos(), rotation.getSin()); + } + + /** + * Checks equality between this Translation2d and another object. + * + * @param obj The other object. + * @return Whether the two objects are equal or not. + */ + @Override + public boolean equals(Object obj) { + if (obj instanceof Translation2d) { + return Math.abs(((Translation2d) obj).m_x - m_x) < 1E-9 + && Math.abs(((Translation2d) obj).m_y - m_y) < 1E-9; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(m_x, m_y); + } + + static class TranslationSerializer extends StdSerializer { + TranslationSerializer() { + super(Translation2d.class); + } + + @Override + public void serialize( + Translation2d value, JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException { + + jgen.writeStartObject(); + jgen.writeNumberField("x", value.m_x); + jgen.writeNumberField("y", value.m_y); + jgen.writeEndObject(); + } + } + + static class TranslationDeserializer extends StdDeserializer { + TranslationDeserializer() { + super(Translation2d.class); + } + + @Override + public Translation2d deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + double xval = node.get("x").numberValue().doubleValue(); + double yval = node.get("y").numberValue().doubleValue(); + + return new Translation2d(xval, yval); + } + } +} diff --git a/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Twist2d.java b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Twist2d.java new file mode 100644 index 000000000..24c8c63c0 --- /dev/null +++ b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/geometry/Twist2d.java @@ -0,0 +1,76 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) 2019 FIRST. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +package edu.wpi.first.wpilibj.geometry; + +import java.util.Objects; + +/** + * A change in distance along arc since the last pose update. We can use ideas + * from differential calculus to create new Pose2ds from a Twist2d and vise + * versa. + * + *

A Twist can be used to represent a difference between two poses. + */ +@SuppressWarnings("MemberName") +public class Twist2d { + /** + * Linear "dx" component. + */ + public double dx; + + /** + * Linear "dy" component. + */ + public double dy; + + /** + * Angular "dtheta" component (radians). + */ + public double dtheta; + + public Twist2d() { + } + + /** + * Constructs a Twist2d with the given values. + * @param dx Change in x direction relative to robot. + * @param dy Change in y direction relative to robot. + * @param dtheta Change in angle relative to robot. + */ + public Twist2d(double dx, double dy, double dtheta) { + this.dx = dx; + this.dy = dy; + this.dtheta = dtheta; + } + + @Override + public String toString() { + return String.format("Twist2d(dX: %.2f, dY: %.2f, dTheta: %.2f)", dx, dy, dtheta); + } + + /** + * Checks equality between this Twist2d and another object. + * + * @param obj The other object. + * @return Whether the two objects are equal or not. + */ + @Override + public boolean equals(Object obj) { + if (obj instanceof Twist2d) { + return Math.abs(((Twist2d) obj).dx - dx) < 1E-9 + && Math.abs(((Twist2d) obj).dy - dy) < 1E-9 + && Math.abs(((Twist2d) obj).dtheta - dtheta) < 1E-9; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(dx, dy, dtheta); + } +} diff --git a/chameleon-server/src/main/java/edu/wpi/first/wpilibj/util/Units.java b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/util/Units.java new file mode 100644 index 000000000..0bdfdc7b6 --- /dev/null +++ b/chameleon-server/src/main/java/edu/wpi/first/wpilibj/util/Units.java @@ -0,0 +1,104 @@ +/*----------------------------------------------------------------------------*/ +/* Copyright (c) 2019 FIRST. All Rights Reserved. */ +/* Open Source Software - may be modified and shared by FRC teams. The code */ +/* must be accompanied by the FIRST BSD license file in the root directory of */ +/* the project. */ +/*----------------------------------------------------------------------------*/ + +package edu.wpi.first.wpilibj.util; + +/** + * Utility class that converts between commonly used units in FRC. + */ +public final class Units { + private static final double kInchesPerFoot = 12.0; + private static final double kMetersPerInch = 0.0254; + private static final double kSecondsPerMinute = 60; + + /** + * Utility class, so constructor is private. + */ + private Units() { + throw new UnsupportedOperationException("This is a utility class!"); + } + + /** + * Converts given meters to feet. + * + * @param meters The meters to convert to feet. + * @return Feet converted from meters. + */ + public static double metersToFeet(double meters) { + return metersToInches(meters) / kInchesPerFoot; + } + + /** + * Converts given feet to meters. + * + * @param feet The feet to convert to meters. + * @return Meters converted from feet. + */ + public static double feetToMeters(double feet) { + return inchesToMeters(feet * kInchesPerFoot); + } + + /** + * Converts given meters to inches. + * + * @param meters The meters to convert to inches. + * @return Inches converted from meters. + */ + public static double metersToInches(double meters) { + return meters / kMetersPerInch; + } + + /** + * Converts given inches to meters. + * + * @param inches The inches to convert to meters. + * @return Meters converted from inches. + */ + public static double inchesToMeters(double inches) { + return inches * kMetersPerInch; + } + + /** + * Converts given degrees to radians. + * + * @param degrees The degrees to convert to radians. + * @return Radians converted from degrees. + */ + public static double degreesToRadians(double degrees) { + return Math.toRadians(degrees); + } + + /** + * Converts given radians to degrees. + * + * @param radians The radians to convert to degrees. + * @return Degrees converted from radians. + */ + public static double radiansToDegrees(double radians) { + return Math.toDegrees(radians); + } + + /** + * Converts rotations per minute to radians per second. + * + * @param rpm The rotations per minute to convert to radians per second. + * @return Radians per second converted from rotations per minute. + */ + public static double rotationsPerMinuteToRadiansPerSecond(double rpm) { + return rpm * Math.PI / (kSecondsPerMinute / 2); + } + + /** + * Converts radians per second to rotations per minute. + * + * @param radiansPerSecond The radians per second to convert to from rotations per minute. + * @return Rotations per minute converted from radians per second. + */ + public static double radiansPerSecondToRotationsPerMinute(double radiansPerSecond) { + return radiansPerSecond * (kSecondsPerMinute / 2) / Math.PI; + } +} diff --git a/chameleon-server/src/main/resources/web/img/chessboard.png b/chameleon-server/src/main/resources/web/img/chessboard.png new file mode 100644 index 000000000..39bb399e8 Binary files /dev/null and b/chameleon-server/src/main/resources/web/img/chessboard.png differ diff --git a/chameleon-server/src/test/java/com/chameleonvision/config/StaticCaptureTest.java b/chameleon-server/src/test/java/com/chameleonvision/config/StaticCaptureTest.java index 267a93220..1856e9f19 100644 --- a/chameleon-server/src/test/java/com/chameleonvision/config/StaticCaptureTest.java +++ b/chameleon-server/src/test/java/com/chameleonvision/config/StaticCaptureTest.java @@ -1,10 +1,12 @@ package com.chameleonvision.config; import com.chameleonvision.util.ProgramDirectoryUtilities; +import com.chameleonvision.vision.camera.CameraStreamer; import com.chameleonvision.vision.image.StaticImageCapture; -import com.chameleonvision.vision.pipeline.CVPipeline2d; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; import edu.wpi.cscore.CameraServerCvJNI; import edu.wpi.cscore.CameraServerJNI; +import edu.wpi.first.networktables.NetworkTableInstance; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -15,7 +17,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -55,12 +56,23 @@ class StaticCaptureTest { } @Test - void ImageProcessTest() { + void ImageProcessTest() throws InterruptedException { ImageLoadTest(); - CVPipeline2d testPipeline = new CVPipeline2d(); + StandardCVPipeline testPipeline = new StandardCVPipeline(); String testImage1 = "CargoSideStraightDark36in"; StaticImageCapture testCapture1 = loadedImages.get(testImage1); testPipeline.initPipeline(testCapture1); + + var streamer = new CameraStreamer(testCapture1, "CargoSideStraightDark36in",testPipeline.settings.streamDivisor); + + NetworkTableInstance.getDefault().startClient("localhost"); + + while(true) { + var result = testPipeline.runPipeline(testCapture1.getFrame().getKey()); + streamer.runStream(result.outputMat); + Thread.sleep(20); + } + } } \ No newline at end of file diff --git a/chameleon-server/src/test/java/com/chameleonvision/scripting/ScriptingTest.java b/chameleon-server/src/test/java/com/chameleonvision/scripting/ScriptingTest.java new file mode 100644 index 000000000..deda0cd12 --- /dev/null +++ b/chameleon-server/src/test/java/com/chameleonvision/scripting/ScriptingTest.java @@ -0,0 +1,29 @@ +package com.chameleonvision.scripting; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import static com.chameleonvision.scripting.ScriptManager.*; + +public class ScriptingTest { + + @Test + public void configTest() { + ScriptConfigManager.deleteConfig(); + + Assertions.assertFalse(ScriptConfigManager.fileExists()); + + ScriptConfigManager.initialize(); + + Assertions.assertTrue(ScriptConfigManager.fileExists()); + + var config = ScriptConfigManager.loadConfig(); + Assertions.assertEquals(config.size(), ScriptEventType.values().length); + System.out.println("Script Config PASSED"); + } + + @Test + public void eventTest() { + ScriptManager.queueEvent(ScriptEventType.kProgramInit); + } +} diff --git a/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/20in.png b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/20in.png new file mode 100644 index 000000000..46cf8c92b Binary files /dev/null and b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/20in.png differ diff --git a/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/30deg40in.png b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/30deg40in.png new file mode 100644 index 000000000..068a1ad86 Binary files /dev/null and b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/30deg40in.png differ diff --git a/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/40deg20in.png b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/40deg20in.png new file mode 100644 index 000000000..819998dc2 Binary files /dev/null and b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/40deg20in.png differ diff --git a/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/40in.png b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/40in.png new file mode 100644 index 000000000..f329e121e Binary files /dev/null and b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/40in.png differ diff --git a/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/SolvePNPtest.java b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/SolvePNPtest.java new file mode 100644 index 000000000..04e8b3317 --- /dev/null +++ b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/SolvePNPtest.java @@ -0,0 +1,38 @@ +package com.chameleonvision.vision.pipeline; + +import com.chameleonvision.vision.image.StaticImageCapture; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipeline; +import com.chameleonvision.vision.pipeline.impl.StandardCVPipelineSettings; +import edu.wpi.cscore.CameraServerCvJNI; +import edu.wpi.cscore.CameraServerJNI; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; + +public class SolvePNPtest { + + private static final Path root = Path.of("src", "test", "java", "com", "chameleonvision", "vision", "pipeline"); + + @Test public void test20in() { + + try { + forceLoad(); + } catch (IOException e) { + return; + } + + // mock up pipeline + var pipeline = new StandardCVPipeline(); + var capture = new StaticImageCapture(Path.of(root.toString(), "20in.png")); + pipeline.initPipeline(capture); + var settings = new StandardCVPipelineSettings(); + + } + + private void forceLoad() throws IOException { + CameraServerJNI.forceLoad(); + CameraServerCvJNI.forceLoad(); + } + +} diff --git a/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/stream.png b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/stream.png new file mode 100644 index 000000000..9bc26d701 Binary files /dev/null and b/chameleon-server/src/test/java/com/chameleonvision/vision/pipeline/stream.png differ