Add GPU Acceleration on the Raspberry Pi (#140)

* Add native stuff

* use runtimeloader

* add more native methods

* more stuff

* Switch JNI methods to static

* Remove non-java classes from the picam jni

* Add gradle task for JNI generation

* Migrate my previous GPU accel work

* Initial work on defining JNI interface

* Change libpicam to a symlink for now

* Initial work on adding no-copy OMX GPU accel on the pi

* Make DIRECT_OMX GPU accel mode not crash

* Clean up OMX changes (still not getting valid data back)

* Re-add GPU unit test

* A couple debugging tweaks/notes

* Add temporary special cases to get RGB out of ProcessingMode.NONE

* Code clarity improvements; fix possible VBO bug

* Get DIRECT_OMX working

* Remove some debugging switches in GPUAccelerator

* Pipe in VCSM stuff to read out pixels FAST

* Apply Spotless

* Revert versioningHelper changes

* Add missing import

* Convert to MMAL and move everything to native

* Re-add shared object

* Rework to use MMAL and do everything natively

* Condense pipeline settings classes

* Add OutputStreamPipeline

* Apply spotless

* Fix duplicate variable inits and add more video modes

* Integrate color frames and latency measurements for GPU

* Fix camera detection on pi and other platforms

* Add proper color copy disabling and camera settings calls

* Fix things that were broken by rebase

* Fix spotless issues and remove uneeded prints

* Remove libpicam symlink

* Fix stream resolution limiting

* Remove testing code in GPUAcceleratedHSVPipe

* Make profiling options general to all computers

* Make PicamJNI load from resources

* run spotlessApply

* Address review comments

* Update Maven repo for JOGL

* Fix release race condition

* Only run GPU accel test on the pi

* Lint fix and merge conflict accident fixes

* Make Jackson ignore extra fields when unmarshalling HardwareConfig

* Fix Mat releasing data race

* Spotless apply

* Remove broken header generation task

* Fix shared library loading typo

* Add a ZeroCopyPicam quirk to allow setting gain with the MMAL backend

* Make sure that exposure/brightness/gain get set after res changes

* Make rawInputMat properly local

* Remove bogus set of shouldRun flag

* Clean up small GPUHSVPipe print

* Add in some things that missed the ZeroCopyPiCameraSource rename

* Fix incorrect scoping introduced in past rebase

* Don't filter out too-low resolutions

* Only show latency when GPU accel is enabled

* Don't free Mats in stream thread before we use them

* Fix use-after-free and latency caluclation bugs on USB camera source

* Update libpicam

* Remove unwanted print

* Add libpicam forceLoad in unit test

* Fix streaming during camera calibration

* Fix zerocopy Picam calculation

* Use logger trace method instead of raw prints

* Fix calibration and driver mode pipes with the Picam

Co-authored-by: Matt <matthew.morley.ca@gmail.com>
Co-authored-by: Banks Troutman <btrout.dhrs@gmail.com>

photon-server/src/test/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProviderTest.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2020 Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.frame.provider;
+
+import java.io.IOException;
+import org.junit.jupiter.api.Test;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.photonvision.common.configuration.CameraConfiguration;
+import org.photonvision.common.util.TestUtils;
+import org.photonvision.raspi.PicamJNI;
+import org.photonvision.vision.camera.ZeroCopyPicamSource;
+
+public class AcceleratedPicamFrameProviderTest {
+    @Test
+    public void testGrabFrame() throws IOException {
+        PicamJNI.forceLoad();
+        if (!PicamJNI.isSupported()) return;
+
+        TestUtils.loadLibraries();
+
+        var frameProvider =
+                new AcceleratedPicamFrameProvider(
+                        new ZeroCopyPicamSource.PicamSettables(new CameraConfiguration("f", "f", "f", "f")));
+
+        long lastTime = System.currentTimeMillis();
+        for (int i = 0; i < 10; i++) {
+            var frame = frameProvider.get();
+            System.out.println(frame.image.getMat().get(0, 0)[0]);
+
+            long time = System.currentTimeMillis();
+            System.out.println("dt (ms): " + (time - lastTime));
+            lastTime = time;
+        }
+        var mat = frameProvider.get().image.getMat();
+        Imgcodecs.imwrite("out.png", mat);
+    }
+}

photon-server/src/test/java/org/photonvision/vision/pipeline/CirclePNPTest.java

@@ -111,7 +111,9 @@ public class CirclePNPTest {
         var frameProvider =
                 new FileFrameProvider(
                         TestUtils.getPowercellImagePath(TestUtils.PowercellTestImages.kPowercell_test_6, false),
-                        TestUtils.WPI2020Image.FOV);
+                        TestUtils.WPI2020Image.FOV,
+                        new Rotation2d(),
+                        TestUtils.get2020LifeCamCoeffs(true));
 
         CVPipelineResult pipelineResult = pipeline.run(frameProvider.get());
         printTestResults(pipelineResult);

photon-server/src/test/java/org/photonvision/vision/pipeline/PipelineProfilerTest.java

@@ -24,7 +24,7 @@ public class PipelineProfilerTest {
 
     @Test
     public void reflectiveProfile() {
-        long[] invalidNanos = new long[10];
+        long[] invalidNanos = new long[20];
         long[] validNanos = new long[PipelineProfiler.ReflectivePipeCount];
 
         for (int i = 0; i < validNanos.length; i++) {
@@ -34,8 +34,9 @@ public class PipelineProfilerTest {
         var invalidResult = PipelineProfiler.getReflectiveProfileString(invalidNanos);
         var validResult = PipelineProfiler.getReflectiveProfileString(validNanos);
 
-        Assertions.assertEquals("Invalid data", invalidResult);
-        Assertions.assertTrue(validResult.contains("Total: 153.0ms"));
         System.out.println(validResult);
+
+        Assertions.assertEquals("Invalid data", invalidResult);
+        Assertions.assertTrue(validResult.contains("Total: 45.0ms"));
     }
 }