From b3436765e114e1905f8a863be3d7a4c0b2bed6b1 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 14 Aug 2020 12:39:21 -0700 Subject: [PATCH] 3d, camera calibration, backend settings hookup (#80) * Implement new UI backend stuff * Kinda partially add resolution accuracy list * camera calibration go brrrrrrrr * ayyyy calibration works * Maybe fix grouping * Reorganize camera view * Fix settings not getting sent * Make pretty (#4) * Reorganize camera view * Apply some cosmetic layout changes to the cameras page * Fix pipeline rollback bug when starting on non-dashboard pages Co-authored-by: Matt * Fix naming mismatch * Mostly make stuff work * rename robot-relative pose to camera-relative pose * SolvePNP memes, fix isFovConfigurable * Change config path to photonvision_config * netmask go poof, fix zip download? * Update index.js * Fix multi cam stuff? * Use LinearFilter instead * Fix multicam * aaa * start adding restart device and restart program, fix square size bug * Add some debug stuff * oop * Start fixing tests * Fix tests * Make target box proportinal * run spotless * Make crosshair h o t p i n k * Address review comments * Address review 2 electric booaloo * Possibly implement vendor FOV? * Make centroid crosshair gren * actually use FOV * Fix tests * actually fix tests Co-authored-by: Declan Freeman-Gleason --- photon-client/src/App.vue | 56 +- .../src/components/common/cv-number-input.vue | 3 +- .../pipeline/CameraAndPipelineSelect.vue | 45 +- photon-client/src/store/index.js | 75 ++- .../src/store/modules/networkSettings.js | 17 - photon-client/src/store/modules/pipeline.js | 45 -- photon-client/src/views/CamerasView.vue | 637 +++++++++++------- photon-client/src/views/PipelineView.vue | 29 +- .../src/views/PipelineViews/PnPTab.vue | 19 +- .../src/views/PipelineViews/TargetsTab.vue | 10 +- photon-client/src/views/SettingsView.vue | 10 +- .../src/views/SettingsViews/General.vue | 82 ++- .../src/views/SettingsViews/Networking.vue | 10 +- photon-server/.gitignore | 3 +- photon-server/build.gradle | 3 + .../edu/wpi/first/wpilibj/LinearFilter.java | 165 +++++ .../edu/wpi/first/wpilibj/MedianFilter.java | 86 --- .../first/wpilibj/geometry/Transform2d.java | 9 + .../src/main/java/org/photonvision/Main.java | 12 +- .../configuration/CameraConfiguration.java | 24 +- .../common/configuration/ConfigManager.java | 104 ++- .../common/configuration/HardwareConfig.java | 70 +- .../common/configuration/NetworkConfig.java | 42 +- .../configuration/PhotonConfiguration.java | 22 +- .../common/dataflow/DataChangeService.java | 1 + .../networktables/NTDataPublisher.java | 10 +- .../networktables/NetworkTablesManager.java | 1 + .../dataflow/websocket/UIDataPublisher.java | 10 +- .../common/hardware/GPIO/GPIOBase.java | 3 +- .../common/hardware/HardwareManager.java | 27 +- .../common/hardware/Platform.java | 15 +- .../metrics/{CPU.java => CPUMetrics.java} | 14 +- .../metrics/{GPU.java => GPUMetrics.java} | 13 +- .../hardware/metrics/MetricsPublisher.java | 24 +- .../metrics/{RAM.java => RAMMetrics.java} | 13 +- .../photonvision/common/logging/Logger.java | 49 +- .../common/networking/LinuxNetworking.java | 3 +- .../common/networking/NetworkInterface.java | 16 +- .../common/networking/NetworkManager.java | 33 + .../common/networking/SysNetworking.java | 2 +- .../common/scripting/ScriptManager.java | 3 +- .../common/util/file/FileUtils.java | 20 + .../common/util/file/JacksonUtils.java | 18 +- .../photonvision/server/RequestHandler.java | 134 +++- .../java/org/photonvision/server/Server.java | 14 +- .../photonvision/server/SocketHandler.java | 32 +- .../server/SocketMessageType.java | 5 +- .../vision/camera/FileVisionSource.java | 13 +- .../vision/camera/USBCameraSource.java | 8 +- .../vision/frame/FrameStaticProperties.java | 18 +- .../frame/provider/FileFrameProvider.java | 20 +- .../vision/pipe/impl/CalculateFPSPipe.java | 44 ++ .../vision/pipe/impl/Calibrate3dPipe.java | 32 +- .../vision/pipe/impl/CornerDetectionPipe.java | 1 - .../vision/pipe/impl/Draw2dTargetsPipe.java | 87 ++- .../pipe/impl/FindBoardCornersPipe.java | 87 +-- .../vision/pipe/impl/SolvePNPPipe.java | 11 +- .../vision/pipeline/CVPipelineSettings.java | 12 +- ...Pipeline.java => Calibrate3dPipeline.java} | 127 ++-- .../Calibration3dPipelineSettings.java | 9 +- .../vision/pipeline/ColoredShapePipeline.java | 7 +- .../vision/pipeline/PipelineType.java | 2 +- .../vision/pipeline/ReflectivePipeline.java | 33 +- .../pipeline/ReflectivePipelineSettings.java | 10 +- .../vision/pipeline/UICalibrationData.java | 89 +++ .../pipeline/result/CVPipelineResult.java | 3 +- .../pipeline/result/SimpleTrackedTarget.java | 27 +- .../vision/processes/PipelineManager.java | 108 ++- .../vision/processes/VisionModule.java | 129 +++- .../vision/processes/VisionModuleManager.java | 7 + .../processes/VisionSourceSettables.java | 41 +- .../vision/target/TargetModel.java | 26 +- .../vision/target/TrackedTarget.java | 16 +- .../src/main/resources/web/index.html | 2 +- .../common/configuration/ConfigTest.java | 31 +- .../hardware/HardwareManagerTest.java | 2 +- .../photonvision/hardware/HardwareTest.java | 24 +- .../vision/pipeline/Calibrate3dPipeTest.java | 56 +- .../vision/pipeline/CirclePNPTest.java | 4 +- .../vision/pipeline/SolvePNPTest.java | 38 +- .../processes/VisionModuleManagerTest.java | 6 +- .../vision/target/TargetCalculationsTest.java | 4 +- .../resources/calibration/lifecam240p.json | 34 - .../resources/calibration/lifecam480p.json | 34 - .../resources/hardware/HardwareConfig.json | 37 +- photon-server/versioningHelper.gradle | 2 +- 86 files changed, 2106 insertions(+), 1173 deletions(-) delete mode 100644 photon-client/src/store/modules/networkSettings.js delete mode 100644 photon-client/src/store/modules/pipeline.js create mode 100644 photon-server/src/main/java/edu/wpi/first/wpilibj/LinearFilter.java delete mode 100644 photon-server/src/main/java/edu/wpi/first/wpilibj/MedianFilter.java rename photon-server/src/main/java/org/photonvision/common/hardware/metrics/{CPU.java => CPUMetrics.java} (81%) rename photon-server/src/main/java/org/photonvision/common/hardware/metrics/{GPU.java => GPUMetrics.java} (79%) rename photon-server/src/main/java/org/photonvision/common/hardware/metrics/{RAM.java => RAMMetrics.java} (78%) create mode 100644 photon-server/src/main/java/org/photonvision/vision/pipe/impl/CalculateFPSPipe.java rename photon-server/src/main/java/org/photonvision/vision/pipeline/{Calibration3dPipeline.java => Calibrate3dPipeline.java} (52%) create mode 100644 photon-server/src/main/java/org/photonvision/vision/pipeline/UICalibrationData.java delete mode 100644 photon-server/src/test/resources/calibration/lifecam240p.json delete mode 100644 photon-server/src/test/resources/calibration/lifecam480p.json diff --git a/photon-client/src/App.vue b/photon-client/src/App.vue index a0411a0a0..ecc6b8aec 100644 --- a/photon-client/src/App.vue +++ b/photon-client/src/App.vue @@ -104,7 +104,9 @@ - {{ $store.state.backendConnected ? "Connected" : "Trying to connect..." }} + + {{ $store.state.backendConnected ? "Connected" : "Trying to connect..." }} + @@ -152,7 +154,7 @@ }, data: () => ({ // Used so that we can switch back to the previously selected pipeline after camera calibration - previouslySelectedIndex: null, + previouslySelectedIndex: undefined, timer: undefined, isLogger: false, log: "", @@ -169,9 +171,9 @@ compact: { get() { if (this.$store.state.compactMode === undefined) { - return this.$vuetify.breakpoint.smAndDown; + return this.$vuetify.breakpoint.smAndDown; } else { - return this.$store.state.compactMode || this.$vuetify.breakpoint.smAndDown; + return this.$store.state.compactMode || this.$vuetify.breakpoint.smAndDown; } }, set(value) { @@ -179,7 +181,7 @@ this.$store.commit("compactMode", value); localStorage.setItem("compactMode", value); }, - } + }, }, created() { document.addEventListener("keydown", e => { @@ -210,16 +212,16 @@ } } } catch (error) { - console.error('error: ' + data.data + " , " + error); + console.error('error: ' + JSON.stringify(data.data) + " , " + error); } }; this.$options.sockets.onopen = () => { - this.$store.state.backendConnected = true; + this.$store.state.backendConnected = true; }; let closed = () => { - this.$store.state.backendConnected = false; - } + this.$store.state.backendConnected = false; + }; this.$options.sockets.onclose = closed; this.$options.sockets.onerror = closed; @@ -228,14 +230,15 @@ methods: { handleMessage(key, value) { if (key === "logMessage") { - console.log("[FROM BACKEND]" + value); - this.logMessage(value, 0); + this.logMessage(value["logMessage"], value["logLevel"]); } else if (key === "updatePipelineResult") { this.$store.commit('mutatePipelineResults', value) } else if (this.$store.state.hasOwnProperty(key)) { this.$store.commit(key, value); } else if (this.$store.getters.currentPipelineSettings.hasOwnProperty(key)) { this.$store.commit('mutatePipeline', {[key]: value}); + } else if (this.$store.state.settings.hasOwnProperty(key)) { + this.$store.commit('mutateSettings', {[key]: value}); } else { switch (key) { default: { @@ -259,9 +262,10 @@ this.timer = setInterval(this.saveSettings, 4000); }, logMessage(message, level) { - const colors = ["\u001b[31m", "\u001b[32m", "\u001b[33m", "\u001b[34m"] + const colors = ["\u001B[30m", "\u001B[31m", "\u001B[33m", "\u001B[32m", "\u001B[37m", "\u001B[36m"] const reset = "\u001b[0m" this.log += `${colors[level]}${message}${reset}\n` + console.log(message) }, switchToDriverMode() { this.previouslySelectedIndex = this.$store.getters.currentPipelineIndex; @@ -269,7 +273,7 @@ }, rollbackPipelineIndex() { if (this.previouslySelectedIndex !== null) { - this.handleInputWithIndex('currentPipeline', this.previouslySelectedIndex) + this.handleInputWithIndex('currentPipeline', this.previouslySelectedIndex || 0); } this.previouslySelectedIndex = null; } @@ -278,7 +282,7 @@ \ No newline at end of file diff --git a/photon-client/src/components/common/cv-number-input.vue b/photon-client/src/components/common/cv-number-input.vue index efbbfe43d..7448a7392 100644 --- a/photon-client/src/components/common/cv-number-input.vue +++ b/photon-client/src/components/common/cv-number-input.vue @@ -21,6 +21,7 @@ type="number" style="width: 70px" :step="step" + :disabled="disabled" :rules="rules" /> @@ -37,7 +38,7 @@ TooltippedLabel, }, // eslint-disable-next-line vue/require-prop-types - props: ['name', 'value', 'step', 'labelCols', 'rules', 'tooltip'], + props: ['name', 'value', 'step', 'labelCols', 'rules', 'tooltip', 'disabled'], computed: { localValue: { get() { diff --git a/photon-client/src/components/pipeline/CameraAndPipelineSelect.vue b/photon-client/src/components/pipeline/CameraAndPipelineSelect.vue index db4ca176b..0909827eb 100644 --- a/photon-client/src/components/pipeline/CameraAndPipelineSelect.vue +++ b/photon-client/src/components/pipeline/CameraAndPipelineSelect.vue @@ -148,22 +148,10 @@ - - @@ -249,11 +237,7 @@ namingDialog: false, newPipelineName: "", duplicateDialog: false, - anotherCamera: false, - pipelineDuplicate: { - pipeline: undefined, - camera: -1 - }, + pipeIndexToDuplicate: undefined } }, computed: { @@ -299,10 +283,10 @@ }, currentPipelineIndex: { get() { - return this.$store.getters.currentPipelineIndex + this.$store.getters.isDriverMode ? 1 : 0; + return this.$store.getters.currentPipelineIndex + (this.$store.getters.isDriverMode ? 1 : 0); }, set(value) { - this.$store.commit('currentPipelineIndex', value - this.$store.getters.isDriverMode ? 1 : 0); + this.$store.commit('currentPipelineIndex', value - (this.$store.getters.isDriverMode ? 1 : 0)); } } }, @@ -332,10 +316,7 @@ this.namingDialog = true; }, openDuplicateDialog() { - this.pipelineDuplicate = { - pipeline: this.currentPipelineIndex - 1, - camera: -1 - }; + this.pipeIndexToDuplicate = this.currentPipelineIndex - 1; this.duplicateDialog = true; }, deleteCurrentPipeline() { @@ -356,19 +337,17 @@ } }, duplicatePipeline() { - if (!this.anotherCamera) { - this.pipelineDuplicate.camera = -1 - } - // this.handleInput("duplicatePipeline", this.pipelineDuplicate); - this.axios.post("http://" + this.$address + "/api/vision/duplicate", this.pipelineDuplicate); + // if (!this.anotherCamera) { + // this.pipelineDuplicate.camera = -1 + // } + this.handleInputWithIndex("duplicatePipeline", this.pipeIndexToDuplicate); + // this.axios.post("http://" + this.$address + "/api/vision/duplicate", this.pipeIndexToDuplicate); + this.closeDuplicateDialog(); }, closeDuplicateDialog() { this.duplicateDialog = false; - this.pipelineDuplicate = { - pipeline: undefined, - camera: -1 - } + this.pipeIndexToDuplicate = undefined; }, discardPipelineNameChange() { this.namingDialog = false; diff --git a/photon-client/src/store/index.js b/photon-client/src/store/index.js index 7b759ef6f..36de173e9 100644 --- a/photon-client/src/store/index.js +++ b/photon-client/src/store/index.js @@ -1,7 +1,6 @@ import Vue from 'vue' import Vuex from 'vuex' -import networkSettings from "./modules/networkSettings" import undoRedo from "./modules/undoRedo"; Vue.use(Vuex); @@ -17,7 +16,6 @@ export default new Vuex.Store({ currentResolutionIndex: 0, }, }, - networkSettings: networkSettings, undoRedo: undoRedo }, state: { @@ -43,6 +41,7 @@ export default new Vuex.Store({ "pixelFormat": "BGR" } ], + calibrations: [ ], fov: 70.0, isFovConfigurable: true, calibrated: false, @@ -76,14 +75,14 @@ export default new Vuex.Store({ solvePNPEnabled: false, targetRegion: 0, contourTargetOrientation: 1, - is3D: false, + + cornerDetectionAccuracyPercentage: 10, // Settings that apply to shape } } ], - pipelineResults: [ - { + pipelineResults: { fps: 0, latency: 0, targets: [{ @@ -95,8 +94,7 @@ export default new Vuex.Store({ // 3D only pose: {x: 0, y: 0, rotation: 0}, }] - } - ], + }, settings: { general: { version: "Unknown", @@ -106,7 +104,7 @@ export default new Vuex.Store({ hardwareModel: "Unknown", hardwarePlatform: "Unknown", }, - networking: { + networkSettings: { teamNumber: 0, supported: true, @@ -120,19 +118,29 @@ export default new Vuex.Store({ supported: true, brightness: 0.0, }, - } + }, + calibrationData: { + count: 0, + videoModeIndex: 0, + minCount: 25, + hasEnough: false, + squareSizeIn: 1.0, + patternWidth: 7, + patternHeight: 7, + boardType: 0, // Chessboard, dotboard + }, }, mutations: { saveBar: set('saveBar'), compactMode: set('compactMode'), cameraSettings: set('cameraSettings'), currentCameraIndex: set('currentCameraIndex'), - pipelineResults: set('pipelineResults'), - networkSettings: set('networkSettings'), selectedOutputs: set('selectedOutputs'), + settings: set('settings'), + calibrationData: set('calibrationData'), - is3D: (state, val) => { - state.cameraSettings[state.currentCameraIndex].currentPipelineSettings.is3D = val; + solvePNPEnabled: (state, val) => { + state.cameraSettings[state.currentCameraIndex].currentPipelineSettings.solvePNPEnabled = val; }, currentPipelineIndex: (state, val) => { @@ -152,27 +160,50 @@ export default new Vuex.Store({ } }, + mutateSettings: (state, payload) => { + for (let key in payload) { + if (!payload.hasOwnProperty(key)) continue; + const value = payload[key]; + const settings = state.settings; + if (settings.hasOwnProperty(key)) { + Vue.set(settings, key, value); + } + } + }, + mutatePipelineResults(state, payload) { // Key: index, value: result - let newResultArray = []; for (let key in payload) { if (!payload.hasOwnProperty(key)) continue; const index = parseInt(key); - newResultArray[index] = payload[key]; + if(index === state.currentCameraIndex) { + Vue.set(state, 'pipelineResults', payload[key]) + } } - Vue.set(state, 'pipelineResults', newResultArray) - } + + }, + + mutateCalibrationState: (state, payload) => { + for (let key in payload) { + if (!payload.hasOwnProperty(key)) continue; + const value = payload[key]; + const calibration = state.calibrationData; + if (calibration.hasOwnProperty(key)) { + calibration[key] = value + } + Vue.set(state, 'calibrationData', calibration) + } + }, }, getters: { isDriverMode: state => state.cameraSettings[state.currentCameraIndex].currentPipelineIndex === -1, - pipelineSettings: state => state.pipelineSettings, streamAddress: state => ["http://" + location.hostname + ":" + state.cameraSettings[state.currentCameraIndex].inputStreamPort + "/stream.mjpg", "http://" + location.hostname + ":" + state.cameraSettings[state.currentCameraIndex].outputStreamPort + "/stream.mjpg"], - targets: state => state.pipelineResults.length, - currentPipelineResults: state => - state.pipelineResults[state.cameraSettings[state.currentCameraIndex].currentPipelineIndex], + currentPipelineResults: state => { + return state.pipelineResults; + }, cameraList: state => state.cameraSettings.map(it => it.nickname), currentCameraSettings: state => state.cameraSettings[state.currentCameraIndex], currentCameraIndex: state => state.currentCameraIndex, @@ -182,6 +213,6 @@ export default new Vuex.Store({ return Object.values(state.cameraSettings[state.currentCameraIndex].videoFormatList); // convert to a list }, pipelineList: state => state.cameraSettings[state.currentCameraIndex].pipelineNicknames, - currentCameraFPS: state => state.pipelineResults[state.currentCameraIndex].fps + calibrationList: state => state.cameraSettings[state.currentCameraIndex].calibrations, } }) \ No newline at end of file diff --git a/photon-client/src/store/modules/networkSettings.js b/photon-client/src/store/modules/networkSettings.js deleted file mode 100644 index a13ba586e..000000000 --- a/photon-client/src/store/modules/networkSettings.js +++ /dev/null @@ -1,17 +0,0 @@ -export default { - state: { - netmask: "", - ip: "", - teamNumber: "", - connectionType: "", - gateway: "" - }, - mutations: { - }, - actions: {}, - getters: { - pipeline: state => { - return state - } - } -}; \ No newline at end of file diff --git a/photon-client/src/store/modules/pipeline.js b/photon-client/src/store/modules/pipeline.js deleted file mode 100644 index 3788a50cf..000000000 --- a/photon-client/src/store/modules/pipeline.js +++ /dev/null @@ -1,45 +0,0 @@ -import Vue from 'vue' - -export default { - state: { - exposure: 0, - brightness: 0, - gain: 0, - rotationMode: 0, - hue: [0, 15], - saturation: [0, 15], - value: [0, 25], - erode: false, - dilate: false, - area: [0, 12], - ratio: [0, 12], - fullness: [0, 12], - speckle: 5, - targetGrouping: 0, - targetIntersection: 0, - sortMode: 0, - multiple: false, - isBinary: 0, - calibrationMode: 0, - videoModeIndex: 0, - streamDivisor: 0, - is3D: false, - targetRegion: 0, - targetOrientation: 1 - }, - mutations: { - isBinary: (state, value) => { - state.isBinary = value - }, - mutatePipeline: (state, {key, value}) => { - Vue.set(state, key, value) - } - - }, - actions: {}, - getters: { - pipeline: state => { - return state - } - } -}; \ No newline at end of file diff --git a/photon-client/src/views/CamerasView.vue b/photon-client/src/views/CamerasView.vue index 7e2666751..6f4f7a850 100644 --- a/photon-client/src/views/CamerasView.vue +++ b/photon-client/src/views/CamerasView.vue @@ -8,6 +8,7 @@ cols="12" md="7" > +
+ + Camera Calibration +
- + + + + + + + + - + + + + + + + + + + + + + + + + + {{ value.width }} X {{ value.height }} + + {{ isCalibrated(value) ? value.mean.toFixed(2) + "px" : "—" }} + + {{ isCalibrated(value) ? value.standardDeviation.toFixed(2) + "px" : "—" }} + + + + + + + Snapshots: {{ snapshotAmount }} of at least {{ minSnapshots }} + + + + + + + + + + + - + - {{ calibrationModeButton.text }} + {{ isCalibrating ? "Take Snapshot" : "Start Calibration" }} - + - {{ cancellationModeButton.text }} + {{ hasEnough ? "End Calibration" : "Cancel Calibration" }} @@ -102,6 +226,7 @@ color="accent" small outlined + style="width: 100%;" @click="downloadBoard" > @@ -113,52 +238,10 @@ ref="calibrationFile" style="color: black; text-decoration: none; display: none" :href="require('../assets/chessboard.png')" - download="Calibration Board.png" + download="chessboard.png" /> - - - Snapshot Amount: {{ snapshotAmount }} - - -
- -
- - - - -
-
@@ -186,207 +269,265 @@ - \ No newline at end of file diff --git a/photon-client/src/views/PipelineView.vue b/photon-client/src/views/PipelineView.vue index fd1023bb4..094943c14 100644 --- a/photon-client/src/views/PipelineView.vue +++ b/photon-client/src/views/PipelineView.vue @@ -25,12 +25,7 @@ class="pb-0 mb-0 pl-4 pt-1" style="height: 15%; min-height: 50px;" > -
- Cameras {{ parseFloat(fps).toFixed(2) }} FPS -
+ Cameras - @@ -166,7 +160,7 @@ slider-color="accent" > {{ tab.name }} @@ -299,11 +293,11 @@ }, processingMode: { get() { - return this.$store.getters.currentPipelineSettings.is3D ? 1 : 0; + return this.$store.getters.currentPipelineSettings.solvePNPEnabled ? 1 : 0; }, set(value) { - this.$store.getters.currentPipelineSettings.is3D = value === 1; - this.handlePipelineUpdate("is3D", value === 1); + this.$store.getters.currentPipelineSettings.solvePNPEnabled = value === 1; + this.handlePipelineUpdate("solvePNPEnabled", value === 1); } }, driverMode: { @@ -349,11 +343,6 @@ // this.handlePipelineUpdate('selectedOutputs', valToCommit); } }, - fps: { - get() { - return this.$store.getters.currentCameraFPS; - } - }, latency: { get() { return this.$store.getters.currentPipelineResults.latency; @@ -384,14 +373,6 @@ height: 100%; } - .fps-indicator { - position: absolute; - top: 2%; - left: 2%; - font-size: 1.75rem; - text-shadow: 1px 1px 5px rgba(1, 1, 1, 0.65); - } - th { width: 80px; text-align: center; diff --git a/photon-client/src/views/PipelineViews/PnPTab.vue b/photon-client/src/views/PipelineViews/PnPTab.vue index 402378ec1..9c41e3060 100644 --- a/photon-client/src/views/PipelineViews/PnPTab.vue +++ b/photon-client/src/views/PipelineViews/PnPTab.vue @@ -22,15 +22,15 @@ @change="onModelSelect" /> Target -