Fix multicam and picam acceleration (#205)

Allows pipelines to access quirks

photon-server/src/main/java/org/photonvision/vision/camera/CameraQuirks.java

@@ -1,22 +0,0 @@
-/*
- * Copyright (C) Photon Vision.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.photonvision.vision.camera;
-
-public enum CameraQuirks {
-    Gain
-}

photon-server/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java

@@ -19,6 +19,7 @@ package org.photonvision.vision.pipeline;
 
 import java.util.List;
 import org.photonvision.common.util.math.MathUtils;
+import org.photonvision.vision.camera.QuirkyCamera;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.frame.FrameStaticProperties;
 import org.photonvision.vision.pipeline.result.CVPipelineResult;
@@ -26,10 +27,13 @@ import org.photonvision.vision.pipeline.result.CVPipelineResult;
 public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelineSettings> {
     protected S settings;
     protected FrameStaticProperties frameStaticProperties;
+    protected QuirkyCamera cameraQuirks;
 
-    protected void setPipeParams(FrameStaticProperties frameStaticProperties, S settings) {
+    protected void setPipeParams(
+            FrameStaticProperties frameStaticProperties, S settings, QuirkyCamera cameraQuirks) {
         this.settings = settings;
         this.frameStaticProperties = frameStaticProperties;
+        this.cameraQuirks = cameraQuirks;
 
         setPipeParamsImpl();
     }
@@ -46,11 +50,11 @@ public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelin
         this.settings = s;
     }
 
-    public R run(Frame frame) {
+    public R run(Frame frame, QuirkyCamera cameraQuirks) {
         if (settings == null) {
             throw new RuntimeException("No settings provided for pipeline!");
         }
-        setPipeParams(frame.frameStaticProperties, settings);
+        setPipeParams(frame.frameStaticProperties, settings, cameraQuirks);
 
         if (frame.image.getMat().empty()) {
             return (R) new CVPipelineResult(0, 0, List.of(), frame);

photon-server/src/main/java/org/photonvision/vision/pipeline/Calibrate3dPipeline.java

@@ -37,6 +37,7 @@ import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.raspi.PicamJNI;
 import org.photonvision.server.SocketHandler;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
+import org.photonvision.vision.camera.CameraQuirk;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
@@ -80,11 +81,6 @@ public class Calibrate3dPipeline
         this.settings = new Calibration3dPipelineSettings();
         this.foundCornersList = new ArrayList<>();
         this.minSnapshots = minSnapshots;
-
-        if (PicamJNI.isSupported()) {
-            PicamJNI.setRotation(settings.inputImageRotationMode.value);
-            PicamJNI.setShouldCopyColor(true);
-        }
     }
 
     @Override
@@ -98,20 +94,24 @@ public class Calibrate3dPipeline
                 new Calibrate3dPipe.CalibratePipeParams(
                         new Size(frameStaticProperties.imageWidth, frameStaticProperties.imageHeight));
         calibrate3dPipe.setParams(calibratePipeParams);
+
+        if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && PicamJNI.isSupported()) {
+            PicamJNI.setRotation(settings.inputImageRotationMode.value);
+            PicamJNI.setShouldCopyColor(true);
+        }
     }
 
     @Override
     protected CVPipelineResult process(Frame frame, Calibration3dPipelineSettings settings) {
         Mat inputColorMat = frame.image.getMat();
-        if (inputColorMat.channels() == 1 && PicamJNI.isSupported()) {
+        if (inputColorMat.channels() == 1
+                && cameraQuirks.hasQuirk(CameraQuirk.PiCam)
+                && PicamJNI.isSupported()) {
             long colorMatPtr = PicamJNI.grabFrame(true);
             if (colorMatPtr == 0) throw new RuntimeException("Got null Mat from GPU Picam driver");
             inputColorMat = new Mat(colorMatPtr);
         }
 
-        // Set the pipe parameters
-        setPipeParams(frame.frameStaticProperties, settings);
-
         if (this.calibrating) {
             return new CVPipelineResult(
                     0, 0, null, new Frame(new CVMat(inputColorMat), frame.frameStaticProperties));

photon-server/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java

@@ -192,8 +192,6 @@ public class ColoredShapePipeline
 
     @Override
     protected CVPipelineResult process(Frame frame, ColoredShapePipelineSettings settings) {
-        setPipeParams(frame.frameStaticProperties, settings);
-
         long sumPipeNanosElapsed = 0L;
 
         var rotateImageResult = rotateImagePipe.run(frame.image.getMat());

photon-server/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java

@@ -22,6 +22,7 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.opencv.core.Mat;
 import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.raspi.PicamJNI;
+import org.photonvision.vision.camera.CameraQuirk;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.pipe.impl.CalculateFPSPipe;
@@ -56,7 +57,7 @@ public class DriverModePipeline
         resizeImagePipe.setParams(
                 new ResizeImagePipe.ResizeImageParams(settings.streamingFrameDivisor));
 
-        if (PicamJNI.isSupported()) {
+        if (PicamJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
             PicamJNI.setRotation(settings.inputImageRotationMode.value);
             PicamJNI.setShouldCopyColor(true);
         }
@@ -65,7 +66,7 @@ public class DriverModePipeline
     @Override
     public DriverModePipelineResult process(Frame frame, DriverModePipelineSettings settings) {
         long totalNanos = 0;
-        boolean accelerated = PicamJNI.isSupported();
+        boolean accelerated = PicamJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam);
 
         // apply pipes
         var inputMat = frame.image.getMat();

photon-server/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java

@@ -21,6 +21,7 @@ import java.util.List;
 import org.opencv.core.Mat;
 import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.raspi.PicamJNI;
+import org.photonvision.vision.camera.CameraQuirk;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.opencv.Contour;
@@ -70,11 +71,7 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
         var rotateImageParams = new RotateImagePipe.RotateImageParams(settings.inputImageRotationMode);
         rotateImagePipe.setParams(rotateImageParams);
 
-        if (!PicamJNI.isSupported()) {
-            var hsvParams =
-                    new HSVPipe.HSVParams(settings.hsvHue, settings.hsvSaturation, settings.hsvValue);
-            hsvPipe.setParams(hsvParams);
-        } else {
+        if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && PicamJNI.isSupported()) {
             PicamJNI.setThresholds(
                     settings.hsvHue.getFirst() / 180d,
                     settings.hsvSaturation.getFirst() / 255d,
@@ -85,6 +82,10 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
 
             PicamJNI.setRotation(settings.inputImageRotationMode.value);
             PicamJNI.setShouldCopyColor(settings.inputShouldShow);
+        } else {
+            var hsvParams =
+                    new HSVPipe.HSVParams(settings.hsvHue, settings.hsvSaturation, settings.hsvValue);
+            hsvPipe.setParams(hsvParams);
         }
 
         var findContoursParams = new FindContoursPipe.FindContoursParams();

photon-server/src/main/java/org/photonvision/vision/processes/VisionModule.java

@@ -86,15 +86,6 @@ public class VisionModule {
                         VisionModule.class,
                         visionSource.getSettables().getConfiguration().nickname,
                         LogGroup.VisionModule);
-        this.pipelineManager = pipelineManager;
-        this.visionSource = visionSource;
-        this.visionRunner =
-                new VisionRunner(
-                        this.visionSource.getFrameProvider(),
-                        this.pipelineManager::getCurrentUserPipeline,
-                        this::consumeResult);
-        this.streamRunnable = new StreamRunnable(new OutputStreamPipeline());
-        this.moduleIndex = index;
 
         // do this
         if (visionSource instanceof USBCameraSource) {
@@ -105,6 +96,17 @@ public class VisionModule {
             cameraQuirks = QuirkyCamera.DefaultCamera;
         }
 
+        this.pipelineManager = pipelineManager;
+        this.visionSource = visionSource;
+        this.visionRunner =
+                new VisionRunner(
+                        this.visionSource.getFrameProvider(),
+                        this.pipelineManager::getCurrentUserPipeline,
+                        this::consumeResult,
+                        this.cameraQuirks);
+        this.streamRunnable = new StreamRunnable(new OutputStreamPipeline());
+        this.moduleIndex = index;
+
         DataChangeService.getInstance().addSubscriber(new VisionModuleChangeSubscriber(this));
 
         createStreams();
@@ -488,7 +490,8 @@ public class VisionModule {
         consumePipelineResult(result);
 
         // Pipelines like DriverMode and Calibrate3dPipeline have null output frames
-        if (result.inputFrame != null) {
+        if (result.inputFrame != null
+                && (pipelineManager.getCurrentPipelineSettings() instanceof AdvancedPipelineSettings)) {
             streamRunnable.updateData(
                     result.inputFrame,
                     result.outputFrame,

photon-server/src/main/java/org/photonvision/vision/processes/VisionRunner.java

@@ -21,6 +21,7 @@ import java.util.function.Consumer;
 import java.util.function.Supplier;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
+import org.photonvision.vision.camera.QuirkyCamera;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.frame.FrameProvider;
 import org.photonvision.vision.pipeline.CVPipeline;
@@ -35,6 +36,7 @@ public class VisionRunner {
     private final Supplier<Frame> frameSupplier;
     private final Supplier<CVPipeline> pipelineSupplier;
     private final Consumer<CVPipelineResult> pipelineResultConsumer;
+    private final QuirkyCamera cameraQuirks;
 
     private long loopCount;
 
@@ -49,10 +51,12 @@ public class VisionRunner {
     public VisionRunner(
             FrameProvider frameSupplier,
             Supplier<CVPipeline> pipelineSupplier,
-            Consumer<CVPipelineResult> pipelineResultConsumer) {
+            Consumer<CVPipelineResult> pipelineResultConsumer,
+            QuirkyCamera cameraQuirks) {
         this.frameSupplier = frameSupplier;
         this.pipelineSupplier = pipelineSupplier;
         this.pipelineResultConsumer = pipelineResultConsumer;
+        this.cameraQuirks = cameraQuirks;
 
         visionProcessThread = new Thread(this::update);
         visionProcessThread.setName("VisionRunner - " + frameSupplier.getName());
@@ -69,7 +73,7 @@ public class VisionRunner {
             var frame = frameSupplier.get();
 
             try {
-                var pipelineResult = pipeline.run(frame);
+                var pipelineResult = pipeline.run(frame, cameraQuirks);
                 pipelineResultConsumer.accept(pipelineResult);
             } catch (Exception ex) {
                 logger.error("Exception on loop " + loopCount);