Add GPU Acceleration on the Raspberry Pi (#140)

* Add native stuff

* use runtimeloader

* add more native methods

* more stuff

* Switch JNI methods to static

* Remove non-java classes from the picam jni

* Add gradle task for JNI generation

* Migrate my previous GPU accel work

* Initial work on defining JNI interface

* Change libpicam to a symlink for now

* Initial work on adding no-copy OMX GPU accel on the pi

* Make DIRECT_OMX GPU accel mode not crash

* Clean up OMX changes (still not getting valid data back)

* Re-add GPU unit test

* A couple debugging tweaks/notes

* Add temporary special cases to get RGB out of ProcessingMode.NONE

* Code clarity improvements; fix possible VBO bug

* Get DIRECT_OMX working

* Remove some debugging switches in GPUAccelerator

* Pipe in VCSM stuff to read out pixels FAST

* Apply Spotless

* Revert versioningHelper changes

* Add missing import

* Convert to MMAL and move everything to native

* Re-add shared object

* Rework to use MMAL and do everything natively

* Condense pipeline settings classes

* Add OutputStreamPipeline

* Apply spotless

* Fix duplicate variable inits and add more video modes

* Integrate color frames and latency measurements for GPU

* Fix camera detection on pi and other platforms

* Add proper color copy disabling and camera settings calls

* Fix things that were broken by rebase

* Fix spotless issues and remove uneeded prints

* Remove libpicam symlink

* Fix stream resolution limiting

* Remove testing code in GPUAcceleratedHSVPipe

* Make profiling options general to all computers

* Make PicamJNI load from resources

* run spotlessApply

* Address review comments

* Update Maven repo for JOGL

* Fix release race condition

* Only run GPU accel test on the pi

* Lint fix and merge conflict accident fixes

* Make Jackson ignore extra fields when unmarshalling HardwareConfig

* Fix Mat releasing data race

* Spotless apply

* Remove broken header generation task

* Fix shared library loading typo

* Add a ZeroCopyPicam quirk to allow setting gain with the MMAL backend

* Make sure that exposure/brightness/gain get set after res changes

* Make rawInputMat properly local

* Remove bogus set of shouldRun flag

* Clean up small GPUHSVPipe print

* Add in some things that missed the ZeroCopyPiCameraSource rename

* Fix incorrect scoping introduced in past rebase

* Don't filter out too-low resolutions

* Only show latency when GPU accel is enabled

* Don't free Mats in stream thread before we use them

* Fix use-after-free and latency caluclation bugs on USB camera source

* Update libpicam

* Remove unwanted print

* Add libpicam forceLoad in unit test

* Fix streaming during camera calibration

* Fix zerocopy Picam calculation

* Use logger trace method instead of raw prints

* Fix calibration and driver mode pipes with the Picam

Co-authored-by: Matt <matthew.morley.ca@gmail.com>
Co-authored-by: Banks Troutman <btrout.dhrs@gmail.com>
This commit is contained in:
Declan Freeman-Gleason
2020-12-08 02:34:21 -05:00
committed by GitHub
parent 15d21b7841
commit c3dbd45716
50 changed files with 1705 additions and 397 deletions

View File

@@ -21,7 +21,6 @@ export default new Vuex.Store({
compactMode: localStorage.getItem("compactMode") === undefined ? undefined : localStorage.getItem("compactMode") === "true", // Compact mode is initially unset on purpose
logMessages: [],
currentCameraIndex: 0,
selectedOutputs: [0, 1], // 0 indicates normal, 1 indicates threshold
cameraSettings: [ // This is a list of objects representing the settings of all cameras
{
tiltDegrees: 0.0,
@@ -65,6 +64,8 @@ export default new Vuex.Store({
contourGroupingMode: 0,
contourIntersection: 0,
contourSortMode: 0,
inputShouldShow: true,
outputShouldShow: true,
outputShouldDraw: true,
outputShowMultipleTargets: false,
offsetRobotOffsetMode: 0,
@@ -144,7 +145,7 @@ export default new Vuex.Store({
metrics: set('metrics'),
logString: (state, newStr) => {
const str = state.logMessages;
str.push(newStr)
str.push(newStr);
Vue.set(state, 'logString', str)
},
@@ -238,6 +239,7 @@ export default new Vuex.Store({
currentCameraIndex: state => state.currentCameraIndex,
currentPipelineIndex: state => state.cameraSettings[state.currentCameraIndex].currentPipelineIndex,
currentPipelineSettings: state => state.cameraSettings[state.currentCameraIndex].currentPipelineSettings,
currentVideoFormat: state => state.cameraSettings[state.currentCameraIndex].videoFormatList[state.cameraSettings[state.currentCameraIndex].currentPipelineSettings.cameraVideoModeIndex],
videoFormatList: state => {
return Object.values(state.cameraSettings[state.currentCameraIndex].videoFormatList); // convert to a list
},

View File

@@ -1,67 +1,69 @@
<template>
<v-card
dark
class="pt-3"
color="primary"
flat
>
<v-card-title>
View Program Logs
<v-card
dark
class="pt-3"
color="primary"
flat
>
<v-card-title>
View Program Logs
<v-btn
color="secondary"
style="margin-left: auto;"
depressed
@click="download('photonlog.log', rawLogs.map(it => it.message).join('\n'))"
>
<v-icon left>
mdi-download
</v-icon>
Download Log
</v-btn>
</v-card-title>
<div class="pr-6 pl-6">
<v-btn-toggle
v-model="logLevel"
dark
multiple
class="fill mb-4"
>
<v-btn
color="secondary"
style="margin-left: auto;"
depressed
@click="download('photonlog.log', rawLogs.map(it => it.message).join('\n'))"
v-for="(level) in possibleLevelArray"
:key="level"
color="secondary"
class="fill"
>
<v-icon left>
mdi-download
</v-icon>
Download Log
{{ level }}
</v-btn>
</v-card-title>
<div class="pr-6 pl-6">
<v-btn-toggle
v-model="logLevel"
dark
multiple
class="fill mb-4"
>
<v-btn
v-for="(level) in possibleLevelArray"
:key="level"
color="secondary"
class="fill"
>
{{ level }}
</v-btn>
</v-btn-toggle>
<!-- Logs -->
</v-btn-toggle>
<!-- Logs -->
<v-virtual-scroll
:items="logMessageArray"
item-height="50"
height="600"
>
<template v-slot="{ item }">
<div :class="[getColor(item) + '--text', 'log-item']">{{ item.message }}</div>
</template>
</v-virtual-scroll>
</div>
<v-virtual-scroll
:items="logMessageArray"
item-height="50"
height="600"
>
<template v-slot="{ item }">
<div :class="[getColor(item) + '--text', 'log-item']">
{{ item.message }}
</div>
</template>
</v-virtual-scroll>
</div>
<v-divider />
<v-divider />
<v-card-actions>
<v-spacer />
<v-btn
color="white"
text
@click="$store.state.logsOverlay = false"
>
Close
</v-btn>
</v-card-actions>
</v-card>
<v-card-actions>
<v-spacer />
<v-btn
color="white"
text
@click="$store.state.logsOverlay = false"
>
Close
</v-btn>
</v-card-actions>
</v-card>
</template>
<script>

View File

@@ -26,6 +26,18 @@
style="height: 15%; min-height: 50px;"
>
Cameras
<v-chip
:class="fpsTooLow ? 'ml-2 mt-1' : 'mt-2'"
x-small
label
:color="fpsTooLow ? 'red' : 'transparent'"
:text-color="fpsTooLow ? 'white' : 'grey'"
>
<span class="pr-1">{{ Math.round($store.state.pipelineResults.fps) }}&nbsp;FPS &ndash;</span>
<span v-if="!fpsTooLow">{{ Math.round($store.state.pipelineResults.latency) }} ms latency</span>
<span v-else-if="!$store.getters.currentPipelineSettings.inputShouldShow">HSV thresholds are too broad; narrow them for better performance</span>
<span v-else>stop viewing the color stream for better performance</span>
</v-chip>
<v-switch
v-model="driverMode"
label="Driver Mode"
@@ -55,7 +67,7 @@
:max-height-md="$store.getters.isDriverMode ? '50vh' : '320px'"
:max-height-xl="$store.getters.isDriverMode ? '60vh' : '450px'"
:alt="'Stream' + idx"
:color-picking="$store.state.colorPicking && idx == 0"
:color-picking="$store.state.colorPicking && idx === 0"
@click="onImageClick"
/>
</div>
@@ -148,7 +160,7 @@
v-for="(tabs, idx) in tabGroups"
:key="idx"
:cols="Math.floor(12 / tabGroups.length)"
:class="idx != tabGroups.length - 1 ? 'pr-3' : ''"
:class="idx !== tabGroups.length - 1 ? 'pr-3' : ''"
align-self="stretch"
>
<v-card
@@ -216,13 +228,7 @@
</v-card-title>
<v-card-text>
Because the current resolution {{
this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex].width
}}
x {{
this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex].height
}}
is not yet calibrated, 3D mode cannot be enabled. Please
Because the current resolution {{ this.$store.getters.currentVideoFormat.width }} x {{ this.$store.getters.currentVideoFormat.height }} is not yet calibrated, 3D mode cannot be enabled. Please
<a
href="/#/cameras"
class="white--text"
@@ -258,7 +264,7 @@ import TargetsTab from "./PipelineViews/TargetsTab";
import PnPTab from './PipelineViews/PnPTab';
export default {
name: 'CameraTab',
name: 'Pipeline',
components: {
CameraAndPipelineSelect,
cvImage,
@@ -334,7 +340,7 @@ export default {
// Three tab groups, one with "input", one with "threshold, contours", and the other with "output, target info, 3D"
ret[0] = [tabs.input];
ret[1] = [tabs.threshold];
ret[2] = [tabs.contours, tabs.output]
ret[2] = [tabs.contours, tabs.output];
ret[3] = [tabs.targets, tabs.pnp];
}
@@ -365,13 +371,15 @@ export default {
// All this logic exists to deal with the reality that the output select buttons sometimes need an array and sometimes need a number (depending on whether or not they're exclusive)
get() {
// We switch the selector to single-select only on sm-and-down size devices, so we have to return a Number instead of an Array in that state
let ret;
let ret = [];
if (this.$store.state.colorPicking) {
ret = [0]; // We want the input stream only while color picking
} else if (!this.$store.getters.isDriverMode) {
ret = this.$store.state.selectedOutputs || [0];
} else if (this.$store.getters.isDriverMode) {
ret = [1]; // We want only the output stream in driver mode
} else {
ret = [1]; // We want the output stream in driver mode
if (this.$store.getters.currentPipelineSettings.inputShouldShow) ret = ret.concat([0]);
if (this.$store.getters.currentPipelineSettings.outputShouldShow) ret = ret.concat([1]);
if (!ret.length) ret = [0];
}
if (this.$vuetify.breakpoint.mdAndUp) {
@@ -383,23 +391,36 @@ export default {
set(value) {
let valToCommit = [0];
if (value instanceof Array) {
// Value is already an array, we don't need to do anything
value.sort(); // Sort for visual consistency
valToCommit = value;
// Value is already an array, we don't need to do anything
valToCommit = value;
} else if (value) {
// Value is assumed to be a number, so we wrap it into an array
valToCommit = [value];
// Value is assumed to be a number, so we wrap it into an array
valToCommit = [value];
}
this.$store.commit("selectedOutputs", valToCommit);
// TODO: Currently the backend just sends both streams regardless of the selected outputs value, so we don't need to send anything
// this.handlePipelineUpdate('selectedOutputs', valToCommit);
this.$store.commit("mutatePipeline", {"inputShouldShow": valToCommit.includes(0)});
this.$store.commit("mutatePipeline", {"outputShouldShow": valToCommit.includes(1)});
this.handlePipelineUpdate("inputShouldShow", valToCommit.includes(0));
}
},
fpsTooLow: {
get() {
// For now we only show the FPS is too low warning when GPU acceleration is enabled, because we don't really trust the presented video modes otherwise
return this.$store.state.pipelineResults.fps - this.$store.getters.currentVideoFormat.fps < -5 && this.$store.state.pipelineResults.fps !== 0 && this.$store.state.settings.general.gpuAcceleration;
}
},
latency: {
get() {
return this.$store.getters.currentPipelineResults.latency;
}
}
},
isCalibrated: {
get() {
const resolution = this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex];
return this.$store.getters.currentCameraSettings.calibrations
.some(e => e.width === resolution.width && e.height === resolution.height)
}
},
},
created() {
this.$store.state.connectedCallbacks.push(this.reloadStreams)
@@ -409,11 +430,6 @@ export default {
// Reload the streams as we technically close and reopen them
this.$refs.streams.forEach(it => it.reload())
},
isCalibrated() {
const resolution = this.$store.getters.videoFormatList[this.$store.getters.currentPipelineSettings.cameraVideoModeIndex];
return this.$store.getters.currentCameraSettings.calibrations
.some(e => e.width === resolution.width && e.height === resolution.height)
},
onImageClick(event) {
// Only run on the input stream
if (event.target.alt !== "Stream0") return;

View File

@@ -56,7 +56,6 @@
tooltip="Resolution to which camera frames are downscaled for streaming to the dashboard"
:list="streamResolutionList"
:select-cols="largeBox"
@input="handlePipelineData('streamingFrameDivisor')"
@rollback="e => rollback('streamingFrameDivisor', e)"
/>
</div>
@@ -66,6 +65,8 @@
import CVslider from '../../components/common/cv-slider'
import CVselect from '../../components/common/cv-select'
const unfilteredStreamDivisors = [1, 2, 4, 6];
export default {
name: 'Input',
components: {
@@ -75,7 +76,9 @@
// eslint-disable-next-line vue/require-prop-types
props: ['value'],
data() {
return {}
return {
rawStreamDivisorIndex: 0,
}
},
computed: {
largeBox: {
@@ -120,18 +123,22 @@
},
cameraVideoModeIndex: {
get() {
return this.$store.getters.currentPipelineSettings.cameraVideoModeIndex
return this.$store.getters.currentPipelineSettings.cameraVideoModeIndex;
},
set(val) {
this.$store.commit("mutatePipeline", {"cameraVideoModeIndex": val});
this.handlePipelineUpdate("streamingFrameDivisor", this.getNumSkippedStreamDivisors());
this.rawStreamDivisorIndex = 0;
}
},
streamingFrameDivisor: {
get() {
return this.$store.getters.currentPipelineSettings.streamingFrameDivisor
return this.rawStreamDivisorIndex;
},
set(val) {
this.$store.commit("mutatePipeline", {"streamingFrameDivisor": val});
this.rawStreamDivisorIndex = val;
this.handlePipelineUpdate("streamingFrameDivisor", this.getNumSkippedStreamDivisors() + val);
}
},
@@ -147,21 +154,31 @@
streamResolutionList: {
get() {
let cam_res = this.$store.getters.videoFormatList[
this.$store.getters.currentCameraSettings.currentPipelineSettings.cameraVideoModeIndex]
const cam_res = this.$store.getters.videoFormatList[
this.$store.getters.currentCameraSettings.currentPipelineSettings.cameraVideoModeIndex];
let tmp_list = [];
tmp_list.push(`${Math.floor(cam_res['width'])} X ${Math.floor(cam_res['height'])}`);
for (let x = 2; x <= 6; x += 2) {
for (const x of this.getRawStreamDivisors()) {
tmp_list.push(`${Math.floor(cam_res['width'] / x)} X ${Math.floor(cam_res['height'] / x)}`);
}
return tmp_list;
}
}
},
methods: {}
methods: {
getRawStreamDivisors() {
// Limit stream res when GPU acceleration is enabled because we *know* that we won't be able to get smooth streams above ~640x480
// It would probably be cleaner if this checked that we're on the Raspi 3 instead of checking for GPU accel status
const width = this.$store.getters.videoFormatList[
this.$store.getters.currentCameraSettings.currentPipelineSettings.cameraVideoModeIndex]['width'];
return unfilteredStreamDivisors.filter((x) => !this.$store.state.settings.general.gpuAcceleration || width / x < 400);
},
getNumSkippedStreamDivisors() {
return unfilteredStreamDivisors.length - this.getRawStreamDivisors().length;
}
}
}
</script>
<style scoped>
</style>
</style>

View File

@@ -53,7 +53,7 @@
import CVslider from '../../components/common/cv-slider'
export default {
name: "SolvePNP",
name: "PnP",
components: {
CVslider,
miniMap

View File

@@ -174,6 +174,7 @@
case 0:
this.currentFunction = undefined;
this.$store.state.colorPicking = false;
this.handlePipelineUpdate("outputShouldDraw", true);
return;
case 1:
@@ -187,7 +188,10 @@
break;
}
this.$store.state.colorPicking = true;
this.handlePipelineUpdate("outputShouldDraw", false);
this.$store.commit("mutatePipeline", {"inputShouldShow": true});
this.handlePipelineUpdate("inputShouldShow", true);
}
}
}

View File

@@ -3,43 +3,88 @@
<v-row class="pa-4">
<table class="infoTable">
<tr>
<th class="infoElem"> Version </th>
<th class="infoElem"> Hardware Model </th>
<th class="infoElem"> Platform </th>
<th class="infoElem"> GPU Acceleration </th>
<th class="infoElem">
Version
</th>
<th class="infoElem">
Hardware Model
</th>
<th class="infoElem">
Platform
</th>
<th class="infoElem">
GPU Acceleration
</th>
</tr>
<tr>
<td class="infoElem">{{ version.replace(" ", "") }}</td>
<td class="infoElem">{{ hwModel.replace(" ", "") }}</td>
<td class="infoElem">{{ platform.replace(" ", "") }}</td>
<td class="infoElem">{{ gpuAccel.replace(" ", "") }}</td>
<td class="infoElem">
{{ version.replace(" ", "") }}
</td>
<td class="infoElem">
{{ hwModel.replace(" ", "") }}
</td>
<td class="infoElem">
{{ platform.replace(" ", "") }}
</td>
<td class="infoElem">
{{ gpuAccel.replace(" ", "") }}
</td>
</tr>
</table>
<table class="infoTable">
<tr>
<th class="infoElem"> CPU Usage </th>
<th class="infoElem"> CPU Temp </th>
<th class="infoElem"> CPU Memory Usage </th>
<th class="infoElem"> GPU Memory Usage </th>
<th class="infoElem"> Disk Usage </th>
<th class="infoElem">
CPU Usage
</th>
<th class="infoElem">
CPU Temp
</th>
<th class="infoElem">
CPU Memory Usage
</th>
<th class="infoElem">
GPU Memory Usage
</th>
<th class="infoElem">
Disk Usage
</th>
</tr>
<tr v-if="metrics.cpuUtil !== 'N/A'">
<td class="infoElem">{{ metrics.cpuUtil.replace(" ", "") }}%</td>
<td class="infoElem">{{ parseInt(metrics.cpuTemp) }}&deg;&nbsp;C</td>
<td class="infoElem">{{ metrics.ramUtil.replace(" ", "") }}MB of {{ metrics.cpuMem }}MB</td>
<td class="infoElem">{{ metrics.gpuMemUtil.replace(" ", "") }}MB of {{ metrics.gpuMem }}MB</td>
<td class="infoElem">{{ metrics.diskUtilPct.replace(" ", "") }}</td>
<td class="infoElem">
{{ metrics.cpuUtil.replace(" ", "") }}%
</td>
<td class="infoElem">
{{ parseInt(metrics.cpuTemp) }}&deg;&nbsp;C
</td>
<td class="infoElem">
{{ metrics.ramUtil.replace(" ", "") }}MB of {{ metrics.cpuMem }}MB
</td>
<td class="infoElem">
{{ metrics.gpuMemUtil.replace(" ", "") }}MB of {{ metrics.gpuMem }}MB
</td>
<td class="infoElem">
{{ metrics.diskUtilPct.replace(" ", "") }}
</td>
</tr>
<tr v-if="metrics.cpuUtil === 'N/A'">
<td class="infoElem">---</td>
<td class="infoElem">---</td>
<td class="infoElem">---</td>
<td class="infoElem">---</td>
<td class="infoElem">---</td>
<td class="infoElem">
---
</td>
<td class="infoElem">
---
</td>
<td class="infoElem">
---
</td>
<td class="infoElem">
---
</td>
<td class="infoElem">
---
</td>
</tr>
</table>
</v-row>
<v-row>