Team2890/PhotonVision
4
0
Fork 0
mirror of https://github.com/PhotonVision/photonvision synced 2026-07-01 02:41:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add GPU Acceleration on the Raspberry Pi (#140)

Browse Source 
* Add native stuff

* use runtimeloader

* add more native methods

* more stuff

* Switch JNI methods to static

* Remove non-java classes from the picam jni

* Add gradle task for JNI generation

* Migrate my previous GPU accel work

* Initial work on defining JNI interface

* Change libpicam to a symlink for now

* Initial work on adding no-copy OMX GPU accel on the pi

* Make DIRECT_OMX GPU accel mode not crash

* Clean up OMX changes (still not getting valid data back)

* Re-add GPU unit test

* A couple debugging tweaks/notes

* Add temporary special cases to get RGB out of ProcessingMode.NONE

* Code clarity improvements; fix possible VBO bug

* Get DIRECT_OMX working

* Remove some debugging switches in GPUAccelerator

* Pipe in VCSM stuff to read out pixels FAST

* Apply Spotless

* Revert versioningHelper changes

* Add missing import

* Convert to MMAL and move everything to native

* Re-add shared object

* Rework to use MMAL and do everything natively

* Condense pipeline settings classes

* Add OutputStreamPipeline

* Apply spotless

* Fix duplicate variable inits and add more video modes

* Integrate color frames and latency measurements for GPU

* Fix camera detection on pi and other platforms

* Add proper color copy disabling and camera settings calls

* Fix things that were broken by rebase

* Fix spotless issues and remove uneeded prints

* Remove libpicam symlink

* Fix stream resolution limiting

* Remove testing code in GPUAcceleratedHSVPipe

* Make profiling options general to all computers

* Make PicamJNI load from resources

* run spotlessApply

* Address review comments

* Update Maven repo for JOGL

* Fix release race condition

* Only run GPU accel test on the pi

* Lint fix and merge conflict accident fixes

* Make Jackson ignore extra fields when unmarshalling HardwareConfig

* Fix Mat releasing data race

* Spotless apply

* Remove broken header generation task

* Fix shared library loading typo

* Add a ZeroCopyPicam quirk to allow setting gain with the MMAL backend

* Make sure that exposure/brightness/gain get set after res changes

* Make rawInputMat properly local

* Remove bogus set of shouldRun flag

* Clean up small GPUHSVPipe print

* Add in some things that missed the ZeroCopyPiCameraSource rename

* Fix incorrect scoping introduced in past rebase

* Don't filter out too-low resolutions

* Only show latency when GPU accel is enabled

* Don't free Mats in stream thread before we use them

* Fix use-after-free and latency caluclation bugs on USB camera source

* Update libpicam

* Remove unwanted print

* Add libpicam forceLoad in unit test

* Fix streaming during camera calibration

* Fix zerocopy Picam calculation

* Use logger trace method instead of raw prints

* Fix calibration and driver mode pipes with the Picam

Co-authored-by: Matt <matthew.morley.ca@gmail.com>
Co-authored-by: Banks Troutman <btrout.dhrs@gmail.com>
This commit is contained in:
Declan Freeman-Gleason
2020-12-08 02:34:21 -05:00
committed by GitHub
parent 15d21b7841
commit c3dbd45716
50 changed files with 1705 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
photon-client/src/views/PipelineView.vue
View File

@@ -26,6 +26,18 @@
style="height: 15%; min-height: 50px;"
>
Cameras
<v-chip
:class="fpsTooLow ? 'ml-2 mt-1' : 'mt-2'"
x-small
label
:color="fpsTooLow ? 'red' : 'transparent'"
:text-color="fpsTooLow ? 'white' : 'grey'"
>
<span class="pr-1">{{ Math.round($store.state.pipelineResults.fps) }}&nbsp;FPS &ndash;</span>
<span v-if="!fpsTooLow">{{ Math.round($store.state.pipelineResults.latency) }} ms latency</span>
<span v-else-if="!$store.getters.currentPipelineSettings.inputShouldShow">HSV thresholds are too broad; narrow them for better performance</span>
<span v-else>stop viewing the color stream for better performance</span>
</v-chip>
<v-switch
v-model="driverMode"
label="Driver Mode"
@@ -55,7 +67,7 @@
:max-height-md="$store.getters.isDriverMode ? '50vh' : '320px'"
:max-height-xl="$store.getters.isDriverMode ? '60vh' : '450px'"
:alt="'Stream' + idx"
:color-picking="$store.state.colorPicking && idx == 0"
:color-picking="$store.state.colorPicking && idx === 0"
@click="onImageClick"
/>
</div>
@@ -148,7 +160,7 @@
v-for="(tabs, idx) in tabGroups"
:key="idx"
:cols="Math.floor(12 / tabGroups.length)"
:class="idx != tabGroups.length - 1 ? 'pr-3' : ''"
:class="idx !== tabGroups.length - 1 ? 'pr-3' : ''"
align-self="stretch"
>
<v-card
@@ -216,13 +228,7 @@
</v-card-title>
<v-card-text>
Because the current resolution {{
this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex].width
}}
x {{
this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex].height
}}
is not yet calibrated, 3D mode cannot be enabled. Please
Because the current resolution {{ this.$store.getters.currentVideoFormat.width }} x {{ this.$store.getters.currentVideoFormat.height }} is not yet calibrated, 3D mode cannot be enabled. Please
<a
href="/#/cameras"
class="white--text"
@@ -258,7 +264,7 @@ import TargetsTab from "./PipelineViews/TargetsTab";
import PnPTab from './PipelineViews/PnPTab';
export default {
name: 'CameraTab',
name: 'Pipeline',
components: {
CameraAndPipelineSelect,
cvImage,
@@ -334,7 +340,7 @@ export default {
// Three tab groups, one with "input", one with "threshold, contours", and the other with "output, target info, 3D"
ret[0] = [tabs.input];
ret[1] = [tabs.threshold];
ret[2] = [tabs.contours, tabs.output]
ret[2] = [tabs.contours, tabs.output];
ret[3] = [tabs.targets, tabs.pnp];
}
@@ -365,13 +371,15 @@ export default {
// All this logic exists to deal with the reality that the output select buttons sometimes need an array and sometimes need a number (depending on whether or not they're exclusive)
get() {
// We switch the selector to single-select only on sm-and-down size devices, so we have to return a Number instead of an Array in that state
let ret;
let ret = [];
if (this.$store.state.colorPicking) {
ret = [0]; // We want the input stream only while color picking
} else if (!this.$store.getters.isDriverMode) {
ret = this.$store.state.selectedOutputs || [0];
} else if (this.$store.getters.isDriverMode) {
ret = [1]; // We want only the output stream in driver mode
} else {
ret = [1]; // We want the output stream in driver mode
if (this.$store.getters.currentPipelineSettings.inputShouldShow) ret = ret.concat([0]);
if (this.$store.getters.currentPipelineSettings.outputShouldShow) ret = ret.concat([1]);
if (!ret.length) ret = [0];
}
if (this.$vuetify.breakpoint.mdAndUp) {
@@ -383,23 +391,36 @@ export default {
set(value) {
let valToCommit = [0];
if (value instanceof Array) {
// Value is already an array, we don't need to do anything
value.sort(); // Sort for visual consistency
valToCommit = value;
// Value is already an array, we don't need to do anything
valToCommit = value;
} else if (value) {
// Value is assumed to be a number, so we wrap it into an array
valToCommit = [value];
// Value is assumed to be a number, so we wrap it into an array
valToCommit = [value];
}
this.$store.commit("selectedOutputs", valToCommit);
// TODO: Currently the backend just sends both streams regardless of the selected outputs value, so we don't need to send anything
// this.handlePipelineUpdate('selectedOutputs', valToCommit);
this.$store.commit("mutatePipeline", {"inputShouldShow": valToCommit.includes(0)});
this.$store.commit("mutatePipeline", {"outputShouldShow": valToCommit.includes(1)});
this.handlePipelineUpdate("inputShouldShow", valToCommit.includes(0));
}
},
fpsTooLow: {
get() {
// For now we only show the FPS is too low warning when GPU acceleration is enabled, because we don't really trust the presented video modes otherwise
return this.$store.state.pipelineResults.fps - this.$store.getters.currentVideoFormat.fps < -5 && this.$store.state.pipelineResults.fps !== 0 && this.$store.state.settings.general.gpuAcceleration;
}
},
latency: {
get() {
return this.$store.getters.currentPipelineResults.latency;
}
}
},
isCalibrated: {
get() {
const resolution = this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex];
return this.$store.getters.currentCameraSettings.calibrations
.some(e => e.width === resolution.width && e.height === resolution.height)
}
},
},
created() {
this.$store.state.connectedCallbacks.push(this.reloadStreams)
@@ -409,11 +430,6 @@ export default {
// Reload the streams as we technically close and reopen them
this.$refs.streams.forEach(it => it.reload())
},
isCalibrated() {
const resolution = this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex];
return this.$store.getters.currentCameraSettings.calibrations
.some(e => e.width === resolution.width && e.height === resolution.height)
},
onImageClick(event) {
// Only run on the input stream
if (event.target.alt !== "Stream0") return;