diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 04df6cdcf..2af036bb8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -375,7 +375,7 @@ jobs: ./gradlew photon-server:shadowJar --max-workers 2 if: ${{ (matrix.arch-override == 'none') }} - # The image will only pull the Pi JAR in + # The image will only pull the Pi32 JAR in - name: Generate image if: ${{ github.event_name != 'pull_request' && (matrix.artifact-name) == 'LinuxArm32' }} run: | diff --git a/build.gradle b/build.gradle index a95bfc2de..fe388df47 100644 --- a/build.gradle +++ b/build.gradle @@ -36,9 +36,10 @@ ext { jniPlatforms = ['linuxarm32'] } else if(project.hasProperty('winonly')) { jniPlatforms = ['windowsx86-64'] + } else if(project.hasProperty('aarch64only')) { + jniPlatforms = ['linuxaarch64bionic'] } else { jniPlatforms = ['linuxarm64', 'linuxarm32', 'linuxx86-64', 'osxuniversal', 'windowsx86-64'] - } println("Building for archs " + jniPlatforms) diff --git a/photon-client/package-lock.json b/photon-client/package-lock.json index b3b6a219c..1f579daa8 100644 --- a/photon-client/package-lock.json +++ b/photon-client/package-lock.json @@ -5,7 +5,6 @@ "requires": true, "packages": { "": { - "name": "photon-client", "version": "3.0.0", "dependencies": { "@femessage/log-viewer": "^1.4.2", @@ -19,7 +18,8 @@ "three-full": "^28.0.2", "vue": "^2.6.12", "vue-axios": "^2.1.5", - "vue-native-websocket": "git+https://github.com/PhotonVision/vue-native-websocket.git#5189f29", "vue-router": "^3.4.3", + "vue-native-websocket": "git+https://git@github.com/PhotonVision/vue-native-websocket.git#5189f29", + "vue-router": "^3.4.3", "vuetify": "^2.3.10", "vuex": "^3.5.1" }, @@ -2682,6 +2682,7 @@ "thread-loader": "^2.1.3", "url-loader": "^2.2.0", "vue-loader": "^15.9.2", + "vue-loader-v16": "npm:vue-loader@^16.0.0-beta.3", "vue-style-loader": "^4.1.2", "webpack": "^4.0.0", "webpack-bundle-analyzer": "^3.8.0", @@ -3115,6 +3116,7 @@ "merge-source-map": "^1.1.0", "postcss": "^7.0.14", "postcss-selector-parser": "^6.0.2", + "prettier": "^1.18.2", "source-map": "~0.6.1", "vue-template-es2015-compiler": "^1.9.0" }, @@ -4701,6 +4703,7 @@ "dependencies": { "anymatch": "~3.1.1", "braces": "~3.0.2", + "fsevents": "~2.1.2", "glob-parent": "~5.1.0", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -9164,6 +9167,9 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "dependencies": { + "graceful-fs": "^4.1.6" + }, "optionalDependencies": { "graceful-fs": "^4.1.6" } @@ -9175,7 +9181,11 @@ "@babel/runtime": "^7.14.0", "atob": "^2.1.2", "btoa": "^1.2.1", - "fflate": "^0.4.8" + "canvg": "^3.0.6", + "core-js": "^3.6.0", + "dompurify": "^2.2.0", + "fflate": "^0.4.8", + "html2canvas": "^1.0.0-rc.5" }, "optionalDependencies": { "canvg": "^3.0.6", @@ -14190,8 +14200,10 @@ "dev": true, "license": "MIT", "dependencies": { + "chokidar": "^3.4.0", "graceful-fs": "^4.1.2", - "neo-async": "^2.5.0" + "neo-async": "^2.5.0", + "watchpack-chokidar2": "^2.0.0" }, "optionalDependencies": { "chokidar": "^3.4.0", @@ -14300,6 +14312,7 @@ "anymatch": "^2.0.0", "async-each": "^1.0.1", "braces": "^2.3.2", + "fsevents": "^1.2.7", "glob-parent": "^3.1.0", "inherits": "^2.0.3", "is-binary-path": "^1.0.0", @@ -14540,6 +14553,7 @@ "anymatch": "^2.0.0", "async-each": "^1.0.1", "braces": "^2.3.2", + "fsevents": "^1.2.7", "glob-parent": "^3.1.0", "inherits": "^2.0.3", "is-binary-path": "^1.0.0", @@ -25270,7 +25284,8 @@ }, "vue-native-websocket": { "version": "git+ssh://git@github.com/PhotonVision/vue-native-websocket.git#7a327918e03b215b6899b0d648c5130ece1fa912", - "from": "vue-native-websocket@git+https://github.com/PhotonVision/vue-native-websocket.git#7a32791" }, + "from": "vue-native-websocket@git+https://git@github.com/PhotonVision/vue-native-websocket.git#5189f29" + }, "vue-router": { "version": "3.4.3" }, diff --git a/photon-client/src/views/CamerasView.vue b/photon-client/src/views/CamerasView.vue index 888b81102..27ad3044e 100644 --- a/photon-client/src/views/CamerasView.vue +++ b/photon-client/src/views/CamerasView.vue @@ -218,6 +218,17 @@ tooltip="Enables or Disables camera automatic adjustment for current lighting conditions" @input="e => handlePipelineUpdate('cameraAutoExposure', e)" /> + Processing @ {{ Math.round($store.state.pipelineResults.fps) }} FPS – - {{ Math.min(Math.round($store.state.pipelineResults.latency), 9999) }} ms latency - HSV thresholds are too broad; narrow them for better performance - stop viewing the raw stream for better performance + HSV thresholds are too broad; narrow them for better performance + stop viewing the raw stream for better performance + {{ Math.min(Math.round($store.state.pipelineResults.latency), 9999) }} ms latency calibrations; @@ -59,15 +62,17 @@ public class CameraConfiguration { public DriverModePipelineSettings driveModeSettings = new DriverModePipelineSettings(); public CameraConfiguration(String baseName, String path) { - this(baseName, baseName, baseName, path); + this(baseName, baseName, baseName, path, new String[0]); } - public CameraConfiguration(String baseName, String uniqueName, String nickname, String path) { + public CameraConfiguration( + String baseName, String uniqueName, String nickname, String path, String[] alternates) { this.baseName = baseName; this.uniqueName = uniqueName; this.nickname = nickname; this.path = path; this.calibrations = new ArrayList<>(); + this.otherPaths = alternates; logger.debug( "Creating USB camera configuration for " @@ -145,4 +150,33 @@ public class CameraConfiguration { .ifPresent(calibrations::remove); calibrations.add(calibration); } + + @Override + public String toString() { + return "CameraConfiguration [baseName=" + + baseName + + ", uniqueName=" + + uniqueName + + ", nickname=" + + nickname + + ", path=" + + path + + ", otherPaths=" + + Arrays.toString(otherPaths) + + ", cameraType=" + + cameraType + + ", FOV=" + + FOV + + ", calibrations=" + + calibrations + + ", currentPipelineIndex=" + + currentPipelineIndex + + ", streamIndex=" + + streamIndex + + ", pipelineSettings=" + + pipelineSettings + + ", driveModeSettings=" + + driveModeSettings + + "]"; + } } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java b/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java index 4230387a9..ba3e75181 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java @@ -25,7 +25,7 @@ import java.util.stream.Collectors; import org.photonvision.PhotonVersion; import org.photonvision.common.hardware.Platform; import org.photonvision.common.util.SerializationUtils; -import org.photonvision.raspi.PicamJNI; +import org.photonvision.raspi.LibCameraJNI; import org.photonvision.vision.processes.VisionModule; import org.photonvision.vision.processes.VisionModuleManager; import org.photonvision.vision.processes.VisionSource; @@ -110,8 +110,8 @@ public class PhotonConfiguration { generalSubmap.put("version", PhotonVersion.versionString); generalSubmap.put( "gpuAcceleration", - PicamJNI.isSupported() - ? "Zerocopy MMAL on " + PicamJNI.getSensorModel().getFriendlyName() + LibCameraJNI.isSupported() + ? "Zerocopy Libcamera on " + LibCameraJNI.getSensorModel().getFriendlyName() : ""); // TODO add support for other types of GPU accel generalSubmap.put("hardwareModel", hardwareConfig.deviceName); generalSubmap.put("hardwarePlatform", Platform.getPlatformName()); diff --git a/photon-core/src/main/java/org/photonvision/common/hardware/PiVersion.java b/photon-core/src/main/java/org/photonvision/common/hardware/PiVersion.java index dbf23ac7e..f0817f70c 100644 --- a/photon-core/src/main/java/org/photonvision/common/hardware/PiVersion.java +++ b/photon-core/src/main/java/org/photonvision/common/hardware/PiVersion.java @@ -34,7 +34,7 @@ public enum PiVersion { private static final ShellExec shell = new ShellExec(true, false); private static final PiVersion currentPiVersion = calcPiVersion(); - PiVersion(String s) { + private PiVersion(String s) { this.identifier = s.toLowerCase(); } diff --git a/photon-core/src/main/java/org/photonvision/common/util/TestUtils.java b/photon-core/src/main/java/org/photonvision/common/util/TestUtils.java index 3bb6296c9..2a50ea5ea 100644 --- a/photon-core/src/main/java/org/photonvision/common/util/TestUtils.java +++ b/photon-core/src/main/java/org/photonvision/common/util/TestUtils.java @@ -330,6 +330,7 @@ public class TestUtils { private static int DefaultTimeoutMillis = 5000; public static void showImage(Mat frame, String title, int timeoutMs) { + if (frame.empty()) return; try { HighGui.imshow(title, frame); HighGui.waitKey(timeoutMs); diff --git a/photon-core/src/main/java/org/photonvision/raspi/LibCameraJNI.java b/photon-core/src/main/java/org/photonvision/raspi/LibCameraJNI.java new file mode 100644 index 000000000..4ebc2c873 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/raspi/LibCameraJNI.java @@ -0,0 +1,191 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.raspi; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; + +public class LibCameraJNI { + private static boolean libraryLoaded = false; + private static Logger logger = new Logger(LibCameraJNI.class, LogGroup.Camera); + + public static final Object CAMERA_LOCK = new Object(); + + public static synchronized void forceLoad() throws IOException { + if (libraryLoaded) return; + + try { + File libDirectory = Path.of("lib/").toFile(); + if (!libDirectory.exists()) { + Files.createDirectory(libDirectory.toPath()).toFile(); + } + + // We always extract the shared object (we could hash each so, but that's a lot of work) + URL resourceURL = LibCameraJNI.class.getResource("/nativelibraries/libphotonlibcamera.so"); + File libFile = Path.of("lib/libphotonlibcamera.so").toFile(); + try (InputStream in = resourceURL.openStream()) { + if (libFile.exists()) Files.delete(libFile.toPath()); + Files.copy(in, libFile.toPath()); + } catch (Exception e) { + logger.error("Could not extract the native library!"); + } + System.load(libFile.getAbsolutePath()); + + libraryLoaded = true; + logger.info("Successfully loaded libpicam shared object"); + } catch (UnsatisfiedLinkError e) { + logger.error("Couldn't load libpicam shared object"); + e.printStackTrace(); + } + } + + public enum SensorModel { + Disconnected, + OV5647, // Picam v1 + IMX219, // Picam v2 + IMX477, // Picam HQ + OV9281, + OV7251, + Unknown; + + public String getFriendlyName() { + switch (this) { + case Disconnected: + return "Disconnected Camera"; + case OV5647: + return "Camera Module v1"; + case IMX219: + return "Camera Module v2"; + case IMX477: + return "HQ Camera"; + case OV9281: + return "OV9281"; + case OV7251: + return "OV7251"; + case Unknown: + default: + return "Unknown Camera"; + } + } + } + + public static SensorModel getSensorModel() { + int model = getSensorModelRaw(); + return SensorModel.values()[model]; + } + + public static boolean isSupported() { + return libraryLoaded + // && getSensorModel() != PicamJNI.SensorModel.Disconnected + // && Platform.isRaspberryPi() + && isLibraryWorking(); + } + + private static native boolean isLibraryWorking(); + + public static native int getSensorModelRaw(); + + // ======================================================== // + + /** + * Creates a new runner with a given width/height/fps + * + * @param width Camera video mode width in pixels + * @param height Camera video mode height in pixels + * @param fps Camera video mode FPS + * @return success of creating a camera object + */ + public static native boolean createCamera(int width, int height, int rotation); + + /** + * Starts the camera thresholder and display threads running. Make sure that this function is + * called syncronously with stopCamera and returnFrame! + */ + public static native boolean startCamera(); + + /** Stops the camera runner. Make sure to call prior to destroying the camera! */ + public static native boolean stopCamera(); + + // Destroy all native resources associated with a camera. Ensure stop is called prior! + public static native boolean destroyCamera(); + + // ======================================================== // + + // Set thresholds on [0..1] + public static native boolean setThresholds( + double hl, double sl, double vl, double hu, double su, double vu, boolean hueInverted); + + public static native boolean setAutoExposure(boolean doAutoExposure); + + // Exposure time, in microseconds + public static native boolean setExposure(int exposureUs); + + // Set brighness on [-1, 1] + public static native boolean setBrightness(double brightness); + + // Unknown ranges for red and blue AWB gain + public static native boolean setAwbGain(double red, double blue); + + /** + * Get the time when the first pixel exposure was started, in the same timebase as libcamera gives + * the frame capture time. Units are nanoseconds. + */ + public static native long getFrameCaptureTime(); + + /** + * Get the current time, in the same timebase as libcamera gives the frame capture time. Units are + * nanoseconds. + */ + public static native long getLibcameraTimestamp(); + + public static native long setFramesToCopy(boolean copyIn, boolean copyOut); + + // Analog gain multiplier to apply to all color channels, on [1, Big Number] + public static native boolean setAnalogGain(double analog); + + /** Block until a new frame is avaliable from native code. */ + public static native boolean awaitNewFrame(); + + /** + * Get a pointer to the most recent color mat generated. Call this immediatly after awaitNewFrame, + * and call onlly once per new frame! + */ + public static native long takeColorFrame(); + + /** + * Get a pointer to the most recent processed mat generated. Call this immediatly after + * awaitNewFrame, and call onlly once per new frame! + */ + public static native long takeProcessedFrame(); + + /** + * Set the GPU processing type we should do. Enum of [none, HSV, greyscale, adaptive threshold]. + */ + public static native boolean setGpuProcessType(int type); + + public static native int getGpuProcessType(); + + // /** Release a frame pointer back to the libcamera driver code to be filled again */ + // public static native long returnFrame(long frame); +} diff --git a/photon-core/src/main/java/org/photonvision/raspi/PicamJNI.java b/photon-core/src/main/java/org/photonvision/raspi/PicamJNI.java deleted file mode 100644 index b987d419b..000000000 --- a/photon-core/src/main/java/org/photonvision/raspi/PicamJNI.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (C) Photon Vision. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.photonvision.raspi; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import org.photonvision.common.hardware.PiVersion; -import org.photonvision.common.hardware.Platform; -import org.photonvision.common.logging.LogGroup; -import org.photonvision.common.logging.Logger; - -public class PicamJNI { - private static boolean libraryLoaded = false; - private static boolean enabled = - false; // TODO once we've sorted out what apriltags needs to be doing, we can bring this back? - private static Logger logger = new Logger(PicamJNI.class, LogGroup.Camera); - - public enum SensorModel { - Disconnected, - OV5647, // Picam v1 - IMX219, // Picam v2 - IMX477, // Picam HQ - Unknown; - - public String getFriendlyName() { - switch (this) { - case Disconnected: - return "Disconnected Camera"; - case OV5647: - return "Camera Module v1"; - case IMX219: - return "Camera Module v2"; - case IMX477: - return "HQ Camera"; - case Unknown: - default: - return "Unknown Camera"; - } - } - } - - public static synchronized void forceLoad() throws IOException { - if (libraryLoaded || !Platform.isRaspberryPi()) return; - - try { - File libDirectory = Path.of("lib/").toFile(); - if (!libDirectory.exists()) { - Files.createDirectory(libDirectory.toPath()).toFile(); - } - - // We always extract the shared object (we could hash each so, but that's a lot of work) - URL resourceURL = PicamJNI.class.getResource("/nativelibraries/libpicam.so"); - File libFile = Path.of("lib/libpicam.so").toFile(); - try (InputStream in = resourceURL.openStream()) { - if (libFile.exists()) Files.delete(libFile.toPath()); - Files.copy(in, libFile.toPath()); - } catch (Exception e) { - logger.error("Could not extract the native library!"); - } - System.load(libFile.getAbsolutePath()); - - libraryLoaded = true; - logger.info("Successfully loaded libpicam shared object"); - } catch (UnsatisfiedLinkError e) { - logger.error("Couldn't load libpicam shared object"); - e.printStackTrace(); - } - } - - public static boolean isSupported() { - var piVer = PiVersion.getPiVersion(); - - boolean hwSupported = - (piVer == PiVersion.PI_3 - || piVer == PiVersion.COMPUTE_MODULE_3 - || piVer == PiVersion.ZERO_2_W); - - return libraryLoaded - && enabled - && isVCSMSupported() - && getSensorModel() != SensorModel.Disconnected - && hwSupported; - } - - public static SensorModel getSensorModel() { - switch (getSensorModelRaw().toLowerCase()) { - case "": - return SensorModel.Disconnected; - case "ov5647": - return SensorModel.OV5647; - case "imx219": - return SensorModel.IMX219; - case "imx477": - return SensorModel.IMX477; - default: - return SensorModel.Unknown; - } - } - - private static native String getSensorModelRaw(); - - // This is the main thing we need that isn't supported on Pi 4s, which makes it a good check - private static native boolean isVCSMSupported(); - - // Everything here is static because multiple picams are unsupported at the hardware level - - /** - * Called once for each video mode change. Starts a native thread running MMAL that stays alive - * until destroyCamera is called. - * - * @return true on error. - */ - public static native boolean createCamera(int width, int height, int fps); - - /** - * Destroys MMAL and EGL contexts. Called once for each video mode change *before* createCamera. - * - * @return true on error. - */ - public static native boolean destroyCamera(); - - public static native void setThresholds( - double hL, double sL, double vL, double hU, double sU, double vU); - - public static native void setInvertHue(boolean shouldInvert); - - public static native boolean setExposure(int exposure); - - public static native boolean setBrightness(int brightness); - - // This adjusts the analog gain (normalized to 0-100); ignores the digital gain - public static native boolean setGain(int gain); - - // Adjusts the auto white balance gains, which are normalized 0-100 in the native code - public static native boolean setAwbGain(int red, int blue); - - public static native boolean setRotation(int rotation); - - public static native void setShouldCopyColor(boolean shouldCopyColor); - - public static native long getFrameLatency(); - - public static native long grabFrame(boolean shouldReturnColor); -} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSettables.java new file mode 100644 index 000000000..33341e72b --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSettables.java @@ -0,0 +1,220 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.camera; + +import edu.wpi.first.cscore.VideoMode; +import edu.wpi.first.math.Pair; +import java.util.HashMap; +import org.photonvision.common.configuration.CameraConfiguration; +import org.photonvision.common.util.math.MathUtils; +import org.photonvision.raspi.LibCameraJNI; +import org.photonvision.vision.camera.LibcameraGpuSource.FPSRatedVideoMode; +import org.photonvision.vision.opencv.ImageRotationMode; +import org.photonvision.vision.processes.VisionSourceSettables; + +public class LibcameraGpuSettables extends VisionSourceSettables { + private FPSRatedVideoMode currentVideoMode; + private double lastExposure = 50; + private int lastBrightness = 50; + private boolean lastExposureMode; + private int lastGain = 50; + private Pair lastAwbGains = new Pair<>(18, 18); + private boolean m_initialized = false; + + private ImageRotationMode m_rotationMode; + + public void setRotation(ImageRotationMode rotationMode) { + if (rotationMode != m_rotationMode) { + m_rotationMode = rotationMode; + + setVideoModeInternal(getCurrentVideoMode()); + } + } + + public LibcameraGpuSettables(CameraConfiguration configuration) { + super(configuration); + + videoModes = new HashMap<>(); + + LibCameraJNI.SensorModel sensorModel = LibCameraJNI.getSensorModel(); + + if (sensorModel == LibCameraJNI.SensorModel.IMX219) { + // Settings for the IMX219 sensor, which is used on the Pi Camera Module v2 + videoModes.put( + 0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 120, 120, .39)); + videoModes.put( + 1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39)); + videoModes.put( + 2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39)); + videoModes.put( + 3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, .39)); + // TODO: fix 1280x720 in the native code and re-add it + videoModes.put( + 4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, .53)); + videoModes.put( + 5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 2, 2464 / 2, 15, 20, 1)); + videoModes.put( + 6, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 4, 2464 / 4, 15, 20, 1)); + } else if (sensorModel == LibCameraJNI.SensorModel.OV9281) { + videoModes.put( + 0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 800, 60, 60, 1)); + videoModes.put( + 1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280 / 2, 800 / 2, 60, 60, 1)); + videoModes.put( + 2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39)); + videoModes.put( + 3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39)); + } else { + if (sensorModel == LibCameraJNI.SensorModel.IMX477) { + LibcameraGpuSource.logger.warn( + "It appears you are using a Pi HQ Camera. This camera is not officially supported. You will have to set your camera FOV differently based on resolution."); + } else if (sensorModel == LibCameraJNI.SensorModel.Unknown) { + LibcameraGpuSource.logger.warn( + "You have an unknown sensor connected to your Pi over CSI! This is likely a bug. If it is not, then you will have to set your camera FOV differently based on resolution."); + } + + // Settings for the OV5647 sensor, which is used by the Pi Camera Module v1 + videoModes.put(0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 90, 90, 1)); + videoModes.put(1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 85, 90, 1)); + videoModes.put( + 2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 960, 720, 45, 49, 0.74)); + // Half the size of the active areas on the OV5647 + videoModes.put( + 3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 2592 / 2, 1944 / 2, 20, 20, 1)); + videoModes.put( + 4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91)); + videoModes.put( + 5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72)); + } + + // TODO need to add more video modes for new sensors here + + currentVideoMode = (FPSRatedVideoMode) videoModes.get(0); + } + + @Override + public double getFOV() { + return getCurrentVideoMode().fovMultiplier * getConfiguration().FOV; + } + + @Override + public void setAutoExposure(boolean cameraAutoExposure) { + lastExposureMode = cameraAutoExposure; + // TODO (Matt) -- call LibCameraJNI's auto exposure function, when that exists + LibCameraJNI.setAutoExposure(cameraAutoExposure); + } + + @Override + public void setExposure(double exposure) { + // Todo (Chris) - for now, handle auto exposure by using -1 + if (exposure < 0.0) { + exposure = -1; + } + + // TODO convert to uS + lastExposure = exposure; + var success = LibCameraJNI.setExposure((int) Math.round(exposure) * 800); + if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera exposure"); + } + + @Override + public void setBrightness(int brightness) { + lastBrightness = brightness; + double realBrightness = MathUtils.map(brightness, 0.0, 100.0, -1.0, 1.0); + var success = LibCameraJNI.setBrightness(realBrightness); + if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera brightness"); + } + + @Override + public void setGain(int gain) { + lastGain = gain; + // TODO units here seem odd -- 5ish seems legit? So divide by 10 + var success = LibCameraJNI.setAnalogGain(gain / 10.0); + if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera gain"); + } + + @Override + public void setRedGain(int red) { + lastAwbGains = Pair.of(red, lastAwbGains.getSecond()); + setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond()); + } + + @Override + public void setBlueGain(int blue) { + lastAwbGains = Pair.of(lastAwbGains.getFirst(), blue); + setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond()); + } + + public void setAwbGain(int red, int blue) { + var success = LibCameraJNI.setAwbGain(red / 10.0, blue / 10.0); + if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera AWB gains"); + } + + @Override + public FPSRatedVideoMode getCurrentVideoMode() { + return currentVideoMode; + } + + @Override + protected void setVideoModeInternal(VideoMode videoMode) { + var mode = (FPSRatedVideoMode) videoMode; + + // We need to make sure that other threads don't try to do anything funny while we're recreating + // the camera + synchronized (LibCameraJNI.CAMERA_LOCK) { + boolean success = false; + if (m_initialized) { + success |= LibCameraJNI.stopCamera(); + success |= LibCameraJNI.destroyCamera(); + } + + // if (!success) { + // throw new RuntimeException( + // "Couldn't destroy a zero copy Pi Camera while switching video modes"); + // } + + System.out.println("Starting camera"); + success |= + LibCameraJNI.createCamera( + mode.width, mode.height, (m_rotationMode == ImageRotationMode.DEG_180 ? 180 : 0)); + success |= LibCameraJNI.startCamera(); + if (!success) { + throw new RuntimeException( + "Couldn't create a zero copy Pi Camera while switching video modes"); + } + m_initialized = true; + } + + // We don't store last settings on the native side, and when you change video mode these get + // reset on MMAL's end + setExposure(lastExposure); + setAutoExposure(lastExposureMode); + setBrightness(lastBrightness); + setGain(lastGain); + setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond()); + + LibCameraJNI.setFramesToCopy(true, true); + + currentVideoMode = mode; + } + + @Override + public HashMap getAllVideoModes() { + return videoModes; + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSource.java b/photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSource.java new file mode 100644 index 000000000..7e4330e4d --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSource.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.camera; + +import edu.wpi.first.cscore.VideoMode; +import org.photonvision.common.configuration.CameraConfiguration; +import org.photonvision.common.configuration.ConfigManager; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.vision.frame.FrameProvider; +import org.photonvision.vision.frame.provider.LibcameraGpuFrameProvider; +import org.photonvision.vision.processes.VisionSource; +import org.photonvision.vision.processes.VisionSourceSettables; + +public class LibcameraGpuSource extends VisionSource { + static final Logger logger = new Logger(LibcameraGpuSource.class, LogGroup.Camera); + + private final LibcameraGpuSettables settables; + private final LibcameraGpuFrameProvider frameProvider; + + public LibcameraGpuSource(CameraConfiguration configuration) { + super(configuration); + if (configuration.cameraType != CameraType.ZeroCopyPicam) { + throw new IllegalArgumentException( + "GPUAcceleratedPicamSource only accepts CameraConfigurations with type Picam"); + } + + settables = new LibcameraGpuSettables(configuration); + frameProvider = new LibcameraGpuFrameProvider(settables); + } + + @Override + public FrameProvider getFrameProvider() { + return frameProvider; + } + + @Override + public VisionSourceSettables getSettables() { + return settables; + } + + /** + * On the OV5649 the actual FPS we want to request from the GPU can be higher than the FPS that we + * can do after processing. On the IMX219 these FPSes match pretty closely, except for the + * 1280x720 mode. We use this to present a rated FPS to the user that's lower than the actual FPS + * we request from the GPU. This is important for setting user expectations, and is also used by + * the frontend to detect and explain FPS drops. This class should ONLY be used by Picam video + * modes! This is to make sure it shows up nice in the frontend + */ + public static class FPSRatedVideoMode extends VideoMode { + public final int fpsActual; + public final double fovMultiplier; + + public FPSRatedVideoMode( + PixelFormat pixelFormat, + int width, + int height, + int ratedFPS, + int actualFPS, + double fovMultiplier) { + super(pixelFormat, width, height, ratedFPS); + + this.fpsActual = actualFPS; + this.fovMultiplier = fovMultiplier; + } + } + + @Override + public boolean isVendorCamera() { + return ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV(); + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/ZeroCopyPicamSource.java b/photon-core/src/main/java/org/photonvision/vision/camera/ZeroCopyPicamSource.java deleted file mode 100644 index c7e980696..000000000 --- a/photon-core/src/main/java/org/photonvision/vision/camera/ZeroCopyPicamSource.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (C) Photon Vision. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.photonvision.vision.camera; - -import edu.wpi.first.cscore.VideoMode; -import edu.wpi.first.math.Pair; -import java.util.HashMap; -import org.photonvision.common.configuration.CameraConfiguration; -import org.photonvision.common.configuration.ConfigManager; -import org.photonvision.common.logging.LogGroup; -import org.photonvision.common.logging.Logger; -import org.photonvision.raspi.PicamJNI; -import org.photonvision.vision.frame.FrameProvider; -import org.photonvision.vision.frame.provider.AcceleratedPicamFrameProvider; -import org.photonvision.vision.processes.VisionSource; -import org.photonvision.vision.processes.VisionSourceSettables; - -public class ZeroCopyPicamSource extends VisionSource { - private static final Logger logger = new Logger(ZeroCopyPicamSource.class, LogGroup.Camera); - - private final VisionSourceSettables settables; - private final AcceleratedPicamFrameProvider frameProvider; - - public ZeroCopyPicamSource(CameraConfiguration configuration) { - super(configuration); - if (configuration.cameraType != CameraType.ZeroCopyPicam) { - throw new IllegalArgumentException( - "GPUAcceleratedPicamSource only accepts CameraConfigurations with type Picam"); - } - - settables = new PicamSettables(configuration); - frameProvider = new AcceleratedPicamFrameProvider(settables); - } - - @Override - public FrameProvider getFrameProvider() { - return frameProvider; - } - - @Override - public VisionSourceSettables getSettables() { - return settables; - } - - /** - * On the OV5649 the actual FPS we want to request from the GPU can be higher than the FPS that we - * can do after processing. On the IMX219 these FPSes match pretty closely, except for the - * 1280x720 mode. We use this to present a rated FPS to the user that's lower than the actual FPS - * we request from the GPU. This is important for setting user expectations, and is also used by - * the frontend to detect and explain FPS drops. This class should ONLY be used by Picam video - * modes! This is to make sure it shows up nice in the frontend - */ - public static class FPSRatedVideoMode extends VideoMode { - public final int fpsActual; - public final double fovMultiplier; - - public FPSRatedVideoMode( - PixelFormat pixelFormat, - int width, - int height, - int ratedFPS, - int actualFPS, - double fovMultiplier) { - super(pixelFormat, width, height, ratedFPS); - - this.fpsActual = actualFPS; - this.fovMultiplier = fovMultiplier; - } - } - - public static class PicamSettables extends VisionSourceSettables { - private FPSRatedVideoMode currentVideoMode; - private double lastExposure = 50; - private int lastBrightness = 50; - private boolean lastExposureMode; - private int lastGain = 50; - private Pair lastAwbGains = new Pair(18, 18); - - public PicamSettables(CameraConfiguration configuration) { - super(configuration); - - videoModes = new HashMap<>(); - PicamJNI.SensorModel sensorModel = PicamJNI.getSensorModel(); - - if (sensorModel == PicamJNI.SensorModel.IMX219) { - // Settings for the IMX219 sensor, which is used on the Pi Camera Module v2 - videoModes.put( - 0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 120, 120, .39)); - videoModes.put( - 1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39)); - videoModes.put( - 2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39)); - videoModes.put( - 3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, .39)); - // TODO: fix 1280x720 in the native code and re-add it - videoModes.put( - 4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, .53)); - } else { - if (sensorModel == PicamJNI.SensorModel.IMX477) { - logger.warn( - "It appears you are using a Pi HQ Camera. This camera is not officially supported. You will have to set your camera FOV differently based on resolution."); - } else if (sensorModel == PicamJNI.SensorModel.Unknown) { - logger.warn( - "You have an unknown sensor connected to your Pi over CSI! This is likely a bug. If it is not, then you will have to set your camera FOV differently based on resolution."); - } - - // Settings for the OV5647 sensor, which is used by the Pi Camera Module v1 - videoModes.put( - 0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 90, 90, 1)); - videoModes.put( - 1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, 1)); - videoModes.put( - 2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 85, 90, 1)); - videoModes.put( - 3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, 1)); - videoModes.put( - 4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 960, 720, 45, 49, 0.74)); - videoModes.put( - 5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91)); - videoModes.put( - 6, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72)); - } - - currentVideoMode = (FPSRatedVideoMode) videoModes.get(0); - } - - @Override - public double getFOV() { - return getCurrentVideoMode().fovMultiplier * getConfiguration().FOV; - } - - @Override - public void setAutoExposure(boolean cameraAutoExposure) { - lastExposureMode = cameraAutoExposure; - // TODO (Matt) -- call PicamJNI's auto exposure function, when that exists - } - - @Override - public void setExposure(double exposure) { - // Todo (Chris) - for now, handle auto exposure by using 100% exposure - if (exposure < 0.0) { - exposure = 100.0; - } - - lastExposure = exposure; - var failure = PicamJNI.setExposure((int) Math.round(exposure)); - if (failure) logger.warn("Couldn't set Pi Camera exposure"); - } - - @Override - public void setBrightness(int brightness) { - lastBrightness = brightness; - var failure = PicamJNI.setBrightness(brightness); - if (failure) logger.warn("Couldn't set Pi Camera brightness"); - } - - @Override - public void setGain(int gain) { - lastGain = gain; - var failure = PicamJNI.setGain(gain); - if (failure) logger.warn("Couldn't set Pi Camera gain"); - } - - @Override - public void setRedGain(int red) { - lastAwbGains = Pair.of(red, lastAwbGains.getSecond()); - setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond()); - } - - @Override - public void setBlueGain(int blue) { - lastAwbGains = Pair.of(lastAwbGains.getFirst(), blue); - setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond()); - } - - public void setAwbGain(int red, int blue) { - var failure = PicamJNI.setAwbGain(red, blue); - if (failure) logger.warn("Couldn't set Pi Camera AWB gains"); - } - - @Override - public FPSRatedVideoMode getCurrentVideoMode() { - return currentVideoMode; - } - - @Override - protected void setVideoModeInternal(VideoMode videoMode) { - var mode = (FPSRatedVideoMode) videoMode; - var failure = PicamJNI.destroyCamera(); - if (failure) - throw new RuntimeException( - "Couldn't destroy a zero copy Pi Camera while switching video modes"); - failure = PicamJNI.createCamera(mode.width, mode.height, mode.fpsActual); - if (failure) - throw new RuntimeException( - "Couldn't create a zero copy Pi Camera while switching video modes"); - - // We don't store last settings on the native side, and when you change video mode these get - // reset on MMAL's end - setExposure(lastExposure); - setAutoExposure(lastExposureMode); - setBrightness(lastBrightness); - setGain(lastGain); - setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond()); - - currentVideoMode = mode; - } - - @Override - public HashMap getAllVideoModes() { - return videoModes; - } - } - - @Override - public boolean isVendorCamera() { - return ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV(); - } -} diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/Frame.java b/photon-core/src/main/java/org/photonvision/vision/frame/Frame.java index b20445099..d4202f77a 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/Frame.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/Frame.java @@ -17,52 +17,58 @@ package org.photonvision.vision.frame; -import org.opencv.core.CvType; -import org.opencv.core.Mat; -import org.opencv.core.Size; import org.photonvision.common.util.math.MathUtils; import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.opencv.Releasable; public class Frame implements Releasable { public final long timestampNanos; - public final CVMat image; + + // Frame should at _least_ contain the thresholded frame, and sometimes the color image + public final CVMat colorImage; + public final CVMat processedImage; + public final FrameThresholdType type; + public final FrameStaticProperties frameStaticProperties; - public Frame(CVMat image, long timestampNanos, FrameStaticProperties frameStaticProperties) { - this.image = image; + public Frame( + CVMat color, + CVMat processed, + FrameThresholdType type, + long timestampNanos, + FrameStaticProperties frameStaticProperties) { + this.colorImage = color; + this.processedImage = processed; + this.type = type; this.timestampNanos = timestampNanos; this.frameStaticProperties = frameStaticProperties; } - public Frame(CVMat image, FrameStaticProperties frameStaticProperties) { - this(image, MathUtils.wpiNanoTime(), frameStaticProperties); + public Frame( + CVMat color, + CVMat processed, + FrameThresholdType processType, + FrameStaticProperties frameStaticProperties) { + this(color, processed, processType, MathUtils.wpiNanoTime(), frameStaticProperties); } public Frame() { - this(new CVMat(), MathUtils.wpiNanoTime(), new FrameStaticProperties(0, 0, 0, null)); - } - - public static Frame emptyFrame(int width, int height) { - return new Frame( - new CVMat(Mat.zeros(new Size(width, height), CvType.CV_8UC3)), + this( + new CVMat(), + new CVMat(), + FrameThresholdType.NONE, MathUtils.wpiNanoTime(), - new FrameStaticProperties(width, height, 0, null)); + new FrameStaticProperties(0, 0, 0, null)); } public void copyTo(Frame destFrame) { - image.getMat().copyTo(destFrame.image.getMat()); - } - - public static Frame copyFromAndRelease(Frame frame) { - var mat = new CVMat(); - frame.image.copyTo(mat); - frame.release(); - return new Frame(mat, frame.timestampNanos, frame.frameStaticProperties); + colorImage.getMat().copyTo(destFrame.colorImage.getMat()); + processedImage.getMat().copyTo(destFrame.processedImage.getMat()); } @Override public void release() { - image.release(); + colorImage.release(); + processedImage.release(); } } diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/FrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/FrameProvider.java index 22b4e178f..3c405632d 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/FrameProvider.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/FrameProvider.java @@ -18,7 +18,21 @@ package org.photonvision.vision.frame; import java.util.function.Supplier; +import org.photonvision.vision.opencv.ImageRotationMode; +import org.photonvision.vision.pipe.impl.HSVPipe; public interface FrameProvider extends Supplier { String getName(); + + /** Ask the camera to produce a certain kind of processed image (eg HSV or greyscale) */ + public void requestFrameThresholdType(FrameThresholdType type); + + /** Ask the camera to rotate frames it outputs */ + public void requestFrameRotation(ImageRotationMode rotationMode); + + /** Ask the camera to provide either the input, output, or both frames. */ + public void requestFrameCopies(boolean copyInput, boolean copyOutput); + + /** Ask the camera to rotate frames it outputs */ + public void requestHsvSettings(HSVPipe.HSVParams params); } diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/NetworkFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/FrameThresholdType.java similarity index 58% rename from photon-core/src/main/java/org/photonvision/vision/frame/provider/NetworkFrameProvider.java rename to photon-core/src/main/java/org/photonvision/vision/frame/FrameThresholdType.java index 26cf1aca3..4e779f3c0 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/provider/NetworkFrameProvider.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/FrameThresholdType.java @@ -15,22 +15,10 @@ * along with this program. If not, see . */ -package org.photonvision.vision.frame.provider; +package org.photonvision.vision.frame; -import org.apache.commons.lang3.NotImplementedException; -import org.photonvision.vision.frame.Frame; -import org.photonvision.vision.frame.FrameProvider; - -public class NetworkFrameProvider implements FrameProvider { - private int count = 0; - - @Override - public Frame get() { - throw new NotImplementedException(""); - } - - @Override - public String getName() { - return "NetworkFrameProvider" + count++; - } +public enum FrameThresholdType { + NONE, + HSV, + GREYSCALE, } diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/FileSaveFrameConsumer.java b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/FileSaveFrameConsumer.java index cd2f25f13..3a1848c9a 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/FileSaveFrameConsumer.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/FileSaveFrameConsumer.java @@ -31,9 +31,9 @@ import org.photonvision.common.dataflow.networktables.NetworkTablesManager; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.common.util.TimedTaskManager; -import org.photonvision.vision.frame.Frame; +import org.photonvision.vision.opencv.CVMat; -public class FileSaveFrameConsumer implements Consumer { +public class FileSaveFrameConsumer implements Consumer { // Formatters to generate unique, timestamped file names private static String FILE_PATH = ConfigManager.getInstance().getImageSavePath().toString(); private static String FILE_EXTENSION = ".jpg"; @@ -62,8 +62,8 @@ public class FileSaveFrameConsumer implements Consumer { this.logger = new Logger(FileSaveFrameConsumer.class, this.camNickname, LogGroup.General); } - public void accept(Frame frame) { - if (frame != null && !frame.image.getMat().empty()) { + public void accept(CVMat image) { + if (image != null && image.getMat() != null && !image.getMat().empty()) { if (lock.tryLock()) { boolean curCommand = entry.get(false); if (curCommand && !prevCommand) { @@ -78,7 +78,7 @@ public class FileSaveFrameConsumer implements Consumer { + tf.format(now) + FILE_EXTENSION; - Imgcodecs.imwrite(savefile, frame.image.getMat()); + Imgcodecs.imwrite(savefile, image.getMat()); // Help the user a bit - set the NT entry back to false after 500ms TimedTaskManager.getInstance().addOneShotTask(this::resetCommand, CMD_RESET_TIME_MS); diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java index 129519675..b070e372d 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java @@ -28,7 +28,7 @@ import org.opencv.core.Point; import org.opencv.core.Rect; import org.opencv.imgproc.Imgproc; import org.photonvision.common.util.ColorHelper; -import org.photonvision.vision.frame.Frame; +import org.photonvision.vision.opencv.CVMat; public class MJPGFrameConsumer { public static final Mat EMPTY_MAT = new Mat(60, 15 * 7, CvType.CV_8UC3); @@ -167,9 +167,9 @@ public class MJPGFrameConsumer { this(name, 320, 240, port); } - public void accept(Frame frame) { - if (frame != null && !frame.image.getMat().empty()) { - cvSource.putFrame(frame.image.getMat()); + public void accept(CVMat image) { + if (image != null && !image.getMat().empty()) { + cvSource.putFrame(image.getMat()); // Make sure our disabled framerate limiting doesn't get confused isDisabled = false; diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProvider.java deleted file mode 100644 index 050fb3fef..000000000 --- a/photon-core/src/main/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProvider.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) Photon Vision. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.photonvision.vision.frame.provider; - -import org.opencv.core.Mat; -import org.photonvision.common.util.math.MathUtils; -import org.photonvision.raspi.PicamJNI; -import org.photonvision.vision.frame.Frame; -import org.photonvision.vision.frame.FrameProvider; -import org.photonvision.vision.opencv.CVMat; -import org.photonvision.vision.processes.VisionSourceSettables; - -public class AcceleratedPicamFrameProvider implements FrameProvider { - private final VisionSourceSettables settables; - - private CVMat mat; - - public AcceleratedPicamFrameProvider(VisionSourceSettables visionSettables) { - this.settables = visionSettables; - - var vidMode = settables.getCurrentVideoMode(); - var failure = PicamJNI.createCamera(vidMode.width, vidMode.height, vidMode.fps); - if (failure) { - failure = PicamJNI.destroyCamera(); - if (failure) throw new RuntimeException("Couldn't destroy Pi camera after init failure!"); - throw new RuntimeException( - "Couldn't initialize zero copy Pi camera; check stdout for native code logs"); - } - } - - @Override - public String getName() { - return "AcceleratedPicamFrameProvider"; - } - - @Override - public Frame get() { - long matHandle = PicamJNI.grabFrame(false); - mat = new CVMat(new Mat(matHandle)); - return new Frame( - mat, - MathUtils.wpiNanoTime() - PicamJNI.getFrameLatency(), - settables.getFrameStaticProperties()); - } -} diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/CpuImageProcessor.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/CpuImageProcessor.java new file mode 100644 index 000000000..f9f28c0a5 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/frame/provider/CpuImageProcessor.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.frame.provider; + +import org.photonvision.common.util.numbers.IntegerCouple; +import org.photonvision.vision.frame.Frame; +import org.photonvision.vision.frame.FrameProvider; +import org.photonvision.vision.frame.FrameStaticProperties; +import org.photonvision.vision.frame.FrameThresholdType; +import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.opencv.ImageRotationMode; +import org.photonvision.vision.pipe.CVPipe.CVPipeResult; +import org.photonvision.vision.pipe.impl.GrayscalePipe; +import org.photonvision.vision.pipe.impl.HSVPipe; +import org.photonvision.vision.pipe.impl.RotateImagePipe; + +public abstract class CpuImageProcessor implements FrameProvider { + protected class CapturedFrame { + CVMat colorImage; + FrameStaticProperties staticProps; + long captureTimestamp; + + public CapturedFrame( + CVMat colorImage, FrameStaticProperties staticProps, long captureTimestampNanos) { + this.colorImage = colorImage; + this.staticProps = staticProps; + this.captureTimestamp = captureTimestampNanos; + } + } + + private final HSVPipe m_hsvPipe = new HSVPipe(); + private final RotateImagePipe m_rImagePipe = new RotateImagePipe(); + private final GrayscalePipe m_grayPipe = new GrayscalePipe(); + FrameThresholdType m_processType; + + private final Object m_mutex = new Object(); + + abstract CapturedFrame getInputMat(); + + public CpuImageProcessor() { + m_hsvPipe.setParams( + new HSVPipe.HSVParams( + new IntegerCouple(0, 180), + new IntegerCouple(0, 255), + new IntegerCouple(0, 255), + false)); + } + + @Override + public final Frame get() { + // TODO Auto-generated method stub + var input = getInputMat(); + + CVMat outputMat = null; + long sumNanos = 0; + + { + CVPipeResult out = m_rImagePipe.run(input.colorImage.getMat()); + sumNanos += out.nanosElapsed; + } + + if (!input.colorImage.getMat().empty()) { + if (m_processType == FrameThresholdType.HSV) { + var hsvResult = m_hsvPipe.run(input.colorImage.getMat()); + outputMat = new CVMat(hsvResult.output); + sumNanos += hsvResult.nanosElapsed; + } else if (m_processType == FrameThresholdType.GREYSCALE) { + var result = m_grayPipe.run(input.colorImage.getMat()); + outputMat = new CVMat(result.output); + sumNanos += result.nanosElapsed; + } else { + outputMat = new CVMat(); + } + } else { + System.out.println("Input was empty!"); + outputMat = new CVMat(); + } + + return new Frame( + input.colorImage, outputMat, m_processType, input.captureTimestamp, input.staticProps); + } + + @Override + public void requestFrameThresholdType(FrameThresholdType type) { + synchronized (m_mutex) { + this.m_processType = type; + } + } + + @Override + public void requestFrameRotation(ImageRotationMode rotationMode) { + synchronized (m_mutex) { + m_rImagePipe.setParams(new RotateImagePipe.RotateImageParams(rotationMode)); + } + } + + /** Ask the camera to rotate frames it outputs */ + public void requestHsvSettings(HSVPipe.HSVParams params) { + synchronized (m_mutex) { + m_hsvPipe.setParams(params); + } + } + + @Override + public void requestFrameCopies(boolean copyInput, boolean copyOutput) { + // We don't actually do zero-copy, so this method is a no-op + return; + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/FileFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/FileFrameProvider.java index aa39bafa0..8c7272ba2 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/provider/FileFrameProvider.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/provider/FileFrameProvider.java @@ -22,8 +22,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import org.opencv.core.Mat; import org.opencv.imgcodecs.Imgcodecs; +import org.photonvision.common.util.math.MathUtils; import org.photonvision.vision.calibration.CameraCalibrationCoefficients; -import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.FrameProvider; import org.photonvision.vision.frame.FrameStaticProperties; import org.photonvision.vision.opencv.CVMat; @@ -32,14 +32,14 @@ import org.photonvision.vision.opencv.CVMat; * A {@link FrameProvider} that will read and provide an image from a {@link java.nio.file.Path * path}. */ -public class FileFrameProvider implements FrameProvider { +public class FileFrameProvider extends CpuImageProcessor { public static final int MAX_FPS = 5; private static int count = 0; private final int thisIndex = count++; private final Path path; private final int millisDelay; - private final Frame originalFrame; + private final CVMat originalFrame; private final FrameStaticProperties properties; @@ -70,7 +70,7 @@ public class FileFrameProvider implements FrameProvider { Mat rawImage = Imgcodecs.imread(path.toString()); if (rawImage.cols() > 0 && rawImage.rows() > 0) { properties = new FrameStaticProperties(rawImage.width(), rawImage.height(), fov, calibration); - originalFrame = new Frame(new CVMat(rawImage), properties); + originalFrame = new CVMat(rawImage); } else { throw new RuntimeException("Image loading failed!"); } @@ -97,9 +97,9 @@ public class FileFrameProvider implements FrameProvider { } @Override - public Frame get() { - Frame outputFrame = new Frame(new CVMat(), properties); - originalFrame.copyTo(outputFrame); + public CapturedFrame getInputMat() { + var out = new CVMat(); + out.copyTo(originalFrame); // block to keep FPS at a defined rate if (System.currentTimeMillis() - lastGetMillis < millisDelay) { @@ -111,7 +111,7 @@ public class FileFrameProvider implements FrameProvider { } lastGetMillis = System.currentTimeMillis(); - return outputFrame; + return new CapturedFrame(out, properties, MathUtils.wpiNanoTime()); } @Override diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/LibcameraGpuFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/LibcameraGpuFrameProvider.java new file mode 100644 index 000000000..7b76f658a --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/frame/provider/LibcameraGpuFrameProvider.java @@ -0,0 +1,114 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.frame.provider; + +import org.opencv.core.Mat; +import org.photonvision.common.util.math.MathUtils; +import org.photonvision.raspi.LibCameraJNI; +import org.photonvision.vision.camera.LibcameraGpuSettables; +import org.photonvision.vision.frame.Frame; +import org.photonvision.vision.frame.FrameProvider; +import org.photonvision.vision.frame.FrameThresholdType; +import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.opencv.ImageRotationMode; +import org.photonvision.vision.pipe.impl.HSVPipe.HSVParams; + +public class LibcameraGpuFrameProvider implements FrameProvider { + private final LibcameraGpuSettables settables; + + public LibcameraGpuFrameProvider(LibcameraGpuSettables visionSettables) { + this.settables = visionSettables; + + var vidMode = settables.getCurrentVideoMode(); + settables.setVideoMode(vidMode); + } + + @Override + public String getName() { + return "AcceleratedPicamFrameProvider"; + } + + int i = 0; + + @Override + public Frame get() { + // We need to make sure that other threads don't try to change video modes while we're waiting + // for a frame + // System.out.println("GET!"); + synchronized (LibCameraJNI.CAMERA_LOCK) { + var success = LibCameraJNI.awaitNewFrame(); + + if (!success) { + System.out.println("No new frame"); + return new Frame(); + } + + var colorMat = new CVMat(new Mat(LibCameraJNI.takeColorFrame())); + var processedMat = new CVMat(new Mat(LibCameraJNI.takeProcessedFrame())); + + // System.out.println("Color mat: " + colorMat.getMat().size()); + + // Imgcodecs.imwrite("color" + i + ".jpg", colorMat.getMat()); + // Imgcodecs.imwrite("processed" + (i) + ".jpg", processedMat.getMat()); + + int itype = LibCameraJNI.getGpuProcessType(); + FrameThresholdType type = FrameThresholdType.NONE; + if (itype < FrameThresholdType.values().length && itype >= 0) { + type = FrameThresholdType.values()[itype]; + } + + var now = LibCameraJNI.getLibcameraTimestamp(); + var capture = LibCameraJNI.getFrameCaptureTime(); + var latency = (now - capture); + + return new Frame( + colorMat, + processedMat, + type, + MathUtils.wpiNanoTime() - latency, + settables.getFrameStaticProperties()); + } + } + + @Override + public void requestFrameThresholdType(FrameThresholdType type) { + LibCameraJNI.setGpuProcessType(type.ordinal()); + } + + @Override + public void requestFrameRotation(ImageRotationMode rotationMode) { + this.settables.setRotation(rotationMode); + } + + @Override + public void requestHsvSettings(HSVParams params) { + LibCameraJNI.setThresholds( + params.getHsvLower().val[0] / 180.0, + params.getHsvLower().val[1] / 255.0, + params.getHsvLower().val[2] / 255.0, + params.getHsvUpper().val[0] / 180.0, + params.getHsvUpper().val[1] / 255.0, + params.getHsvUpper().val[2] / 255.0, + params.getHueInverted()); + } + + @Override + public void requestFrameCopies(boolean copyInput, boolean copyOutput) { + LibCameraJNI.setFramesToCopy(copyInput, copyOutput); + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/USBFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/USBFrameProvider.java index 232e3e301..03459c197 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/provider/USBFrameProvider.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/provider/USBFrameProvider.java @@ -19,12 +19,10 @@ package org.photonvision.vision.frame.provider; import edu.wpi.first.cscore.CvSink; import org.photonvision.common.util.math.MathUtils; -import org.photonvision.vision.frame.Frame; -import org.photonvision.vision.frame.FrameProvider; import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.processes.VisionSourceSettables; -public class USBFrameProvider implements FrameProvider { +public class USBFrameProvider extends CpuImageProcessor { private final CvSink cvSink; @SuppressWarnings("SpellCheckingInspection") @@ -38,18 +36,19 @@ public class USBFrameProvider implements FrameProvider { } @Override - public Frame get() { + public CapturedFrame getInputMat() { var mat = new CVMat(); // We do this so that we don't fill a Mat in use by another thread // This is from wpi::Now, or WPIUtilJNI.now() long time = - cvSink.grabFrame( - mat.getMat()); // Units are microseconds, epoch is the same as the Unix epoch + cvSink.grabFrame(mat.getMat()) + * 1000; // Units are microseconds, epoch is the same as the Unix epoch // Sometimes CSCore gives us a zero frametime. if (time <= 1e-6) { time = MathUtils.wpiNanoTime(); } - return new Frame(mat, MathUtils.microsToNanos(time), settables.getFrameStaticProperties()); + + return new CapturedFrame(mat, settables.getFrameStaticProperties(), time); } @Override diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java index 5f2662754..32ea2938b 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java @@ -20,11 +20,11 @@ package org.photonvision.vision.pipe.impl; import edu.wpi.first.apriltag.AprilTagDetection; import edu.wpi.first.apriltag.AprilTagDetector; import java.util.List; -import org.opencv.core.Mat; +import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.pipe.CVPipe; public class AprilTagDetectionPipe - extends CVPipe, AprilTagDetectionPipeParams> { + extends CVPipe, AprilTagDetectionPipeParams> { private final AprilTagDetector m_detector = new AprilTagDetector(); boolean useNativePoseEst; @@ -37,8 +37,12 @@ public class AprilTagDetectionPipe } @Override - protected List process(Mat in) { - var ret = m_detector.detect(in); + protected List process(CVMat in) { + if (in.getMat().empty()) { + return List.of(); + } + + var ret = m_detector.detect(in.getMat()); if (ret == null) { return List.of(); diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterContoursPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterContoursPipe.java index 8ebd78d03..336dfef7a 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterContoursPipe.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterContoursPipe.java @@ -107,8 +107,8 @@ public class FilterContoursPipe // Fullness Filtering. double contourArea = contour.getArea(); - double minFullness = params.getFullness().getFirst() * minAreaRect.size.area() / 100; - double maxFullness = params.getFullness().getSecond() * minAreaRect.size.area() / 100; + double minFullness = params.getFullness().getFirst() * minAreaRect.size.area() / 100.0; + double maxFullness = params.getFullness().getSecond() * minAreaRect.size.area() / 100.0; if (contourArea <= minFullness || contourArea >= maxFullness) return; // Aspect Ratio Filtering. diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java index 4aa4594ae..e4d1efb99 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java @@ -24,12 +24,9 @@ import edu.wpi.first.math.geometry.Transform3d; import edu.wpi.first.math.util.Units; import java.util.ArrayList; import java.util.List; -import org.opencv.core.Mat; import org.photonvision.common.util.math.MathUtils; -import org.photonvision.raspi.PicamJNI; -import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.frame.Frame; -import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.frame.FrameThresholdType; import org.photonvision.vision.pipe.CVPipe.CVPipeResult; import org.photonvision.vision.pipe.impl.*; import org.photonvision.vision.pipe.impl.AprilTagPoseEstimatorPipe.AprilTagPoseEstimatorPipeParams; @@ -39,17 +36,19 @@ import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters; @SuppressWarnings("DuplicatedCode") public class AprilTagPipeline extends CVPipeline { - private final RotateImagePipe rotateImagePipe = new RotateImagePipe(); - private final GrayscalePipe grayscalePipe = new GrayscalePipe(); private final AprilTagDetectionPipe aprilTagDetectionPipe = new AprilTagDetectionPipe(); private final AprilTagPoseEstimatorPipe poseEstimatorPipe = new AprilTagPoseEstimatorPipe(); private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe(); + private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.GREYSCALE; + public AprilTagPipeline() { + super(PROCESSING_TYPE); settings = new AprilTagPipelineSettings(); } public AprilTagPipeline(AprilTagPipelineSettings settings) { + super(PROCESSING_TYPE); this.settings = settings; } @@ -58,15 +57,11 @@ public class AprilTagPipeline extends CVPipeline grayscalePipeResult; - Mat rawInputMat; - boolean inputSingleChannel = frame.image.getMat().channels() == 1; - - if (inputSingleChannel) { - rawInputMat = new Mat(PicamJNI.grabFrame(true)); - frame.image.getMat().release(); // release the 8bit frame ASAP. - } else { - rawInputMat = frame.image.getMat(); - var rotateImageResult = rotateImagePipe.run(rawInputMat); - sumPipeNanosElapsed += rotateImageResult.nanosElapsed; - } - - var inputFrame = new Frame(new CVMat(rawInputMat), frameStaticProperties); - - grayscalePipeResult = grayscalePipe.run(rawInputMat); - sumPipeNanosElapsed += grayscalePipeResult.nanosElapsed; - - var outputFrame = new Frame(new CVMat(grayscalePipeResult.output), frameStaticProperties); - List targetList; - CVPipeResult> tagDetectionPipeResult; // Use the solvePNP Enabled flag to enable native pose estimation aprilTagDetectionPipe.setNativePoseEstimationEnabled(settings.solvePNPEnabled); - tagDetectionPipeResult = aprilTagDetectionPipe.run(grayscalePipeResult.output); + if (frame.type != FrameThresholdType.GREYSCALE) { + // TODO so all cameras should give us ADAPTIVE_THRESH -- how should we handle if not? + return new CVPipelineResult(0, 0, List.of()); + } + + CVPipeResult> tagDetectionPipeResult; + tagDetectionPipeResult = aprilTagDetectionPipe.run(frame.processedImage); sumPipeNanosElapsed += tagDetectionPipeResult.nanosElapsed; targetList = new ArrayList<>(); @@ -194,6 +174,6 @@ public class AprilTagPipeline extends CVPipeline { @@ -28,6 +28,16 @@ public abstract class CVPipeline(); - hsvPipeResult.output = frame.image.getMat(); - hsvPipeResult.nanosElapsed = MathUtils.wpiNanoTime() - frame.timestampNanos; - - sumPipeNanosElapsed += hsvPipeResult.nanosElapsed; - } - // var erodeDilateResult = erodeDilatePipe.run(rawInputMat); // sumPipeNanosElapsed += erodeDilateResult.nanosElapsed; // // CVPipeResult hsvPipeResult = hsvPipe.run(rawInputMat); // sumPipeNanosElapsed += hsvPipeResult.nanosElapsed; - CVPipeResult> findContoursResult = findContoursPipe.run(hsvPipeResult.output); + CVPipeResult> findContoursResult = + findContoursPipe.run(frame.processedImage.getMat()); sumPipeNanosElapsed += findContoursResult.nanosElapsed; CVPipeResult> speckleRejectResult = @@ -247,7 +192,7 @@ public class ColoredShapePipeline List shapes = null; if (settings.contourShape == ContourShape.Circle) { CVPipeResult> findCirclesResult = - findCirclesPipe.run(Pair.of(hsvPipeResult.output, speckleRejectResult.output)); + findCirclesPipe.run(Pair.of(frame.processedImage.getMat(), speckleRejectResult.output)); sumPipeNanosElapsed += findCirclesResult.nanosElapsed; shapes = findCirclesResult.output; } else { @@ -293,11 +238,6 @@ public class ColoredShapePipeline var fpsResult = calculateFPSPipe.run(null); var fps = fpsResult.output; - return new CVPipelineResult( - sumPipeNanosElapsed, - fps, - targetList, - new Frame(new CVMat(hsvPipeResult.output), frame.frameStaticProperties), - new Frame(new CVMat(rawInputMat), frame.frameStaticProperties)); + return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, frame); } } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java index 844f6db86..a665f6415 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java @@ -19,12 +19,11 @@ package org.photonvision.vision.pipeline; import java.util.List; import org.apache.commons.lang3.tuple.Pair; -import org.opencv.core.Mat; import org.photonvision.common.util.math.MathUtils; -import org.photonvision.raspi.PicamJNI; +import org.photonvision.raspi.LibCameraJNI; import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.frame.Frame; -import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.frame.FrameThresholdType; import org.photonvision.vision.pipe.impl.CalculateFPSPipe; import org.photonvision.vision.pipe.impl.Draw2dCrosshairPipe; import org.photonvision.vision.pipe.impl.ResizeImagePipe; @@ -38,10 +37,18 @@ public class DriverModePipeline private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe(); private final ResizeImagePipe resizeImagePipe = new ResizeImagePipe(); + private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE; + public DriverModePipeline() { + super(PROCESSING_TYPE); settings = new DriverModePipelineSettings(); } + public DriverModePipeline(DriverModePipelineSettings settings) { + super(PROCESSING_TYPE); + this.settings = settings; + } + @Override protected void setPipeParamsImpl() { RotateImagePipe.RotateImageParams rotateImageParams = @@ -56,25 +63,19 @@ public class DriverModePipeline resizeImagePipe.setParams( new ResizeImagePipe.ResizeImageParams(settings.streamingFrameDivisor)); - if (PicamJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam)) { - PicamJNI.setRotation(settings.inputImageRotationMode.value); - PicamJNI.setShouldCopyColor(true); - } + // if (LibCameraJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam)) { + // LibCameraJNI.setRotation(settings.inputImageRotationMode.value); + // LibCameraJNI.setShouldCopyColor(true); + // } } @Override public DriverModePipelineResult process(Frame frame, DriverModePipelineSettings settings) { long totalNanos = 0; - boolean accelerated = PicamJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam); + boolean accelerated = LibCameraJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam); // apply pipes - var inputMat = frame.image.getMat(); - if (inputMat.channels() == 1 && accelerated) { - long colorMatPtr = PicamJNI.grabFrame(true); - if (colorMatPtr == 0) throw new RuntimeException("Got null Mat from GPU Picam driver"); - frame.image.release(); - inputMat = new Mat(colorMatPtr); - } + var inputMat = frame.colorImage.getMat(); totalNanos += resizeImagePipe.run(inputMat).nanosElapsed; @@ -91,9 +92,10 @@ public class DriverModePipeline var fpsResult = calculateFPSPipe.run(null); var fps = fpsResult.output; + // Flip-flop input and output in the Frame return new DriverModePipelineResult( MathUtils.nanosToMillis(totalNanos), fps, - new Frame(new CVMat(inputMat), frame.frameStaticProperties)); + new Frame(frame.processedImage, frame.colorImage, frame.type, frame.frameStaticProperties)); } } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java index 5c0d9849f..8be13f757 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java @@ -21,7 +21,6 @@ import java.util.List; import org.apache.commons.lang3.tuple.Pair; import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.FrameStaticProperties; -import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.opencv.ContourShape; import org.photonvision.vision.opencv.DualOffsetValues; import org.photonvision.vision.pipe.impl.*; @@ -99,18 +98,19 @@ public class OutputStreamPipeline { } public CVPipelineResult process( - Frame inputFrame, - Frame outputFrame, + Frame inputAndOutputFrame, AdvancedPipelineSettings settings, List targetsToDraw) { - setPipeParams(inputFrame.frameStaticProperties, settings); - var inMat = inputFrame.image.getMat(); - var outMat = outputFrame.image.getMat(); + setPipeParams(inputAndOutputFrame.frameStaticProperties, settings); + var inMat = inputAndOutputFrame.colorImage.getMat(); + var outMat = inputAndOutputFrame.processedImage.getMat(); long sumPipeNanosElapsed = 0L; // Resize both in place before doing any conversion - sumPipeNanosElapsed += pipeProfileNanos[0] = resizeImagePipe.run(inMat).nanosElapsed; + boolean inEmpty = inMat.empty(); + if (!inEmpty) + sumPipeNanosElapsed += pipeProfileNanos[0] = resizeImagePipe.run(inMat).nanosElapsed; sumPipeNanosElapsed += pipeProfileNanos[1] = resizeImagePipe.run(outMat).nanosElapsed; // Convert single-channel HSV output mat to 3-channel BGR in preparation for streaming @@ -183,7 +183,6 @@ public class OutputStreamPipeline { sumPipeNanosElapsed, fps, // Unused but here just in case targetsToDraw, - new Frame(new CVMat(outMat), outputFrame.frameStaticProperties), - new Frame(new CVMat(inMat), inputFrame.frameStaticProperties)); + inputAndOutputFrame); } } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java index 8372904d4..cea2e3142 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java @@ -18,12 +18,8 @@ package org.photonvision.vision.pipeline; import java.util.List; -import org.opencv.core.Mat; -import org.photonvision.common.util.math.MathUtils; -import org.photonvision.raspi.PicamJNI; -import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.frame.Frame; -import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.frame.FrameThresholdType; import org.photonvision.vision.opencv.Contour; import org.photonvision.vision.opencv.DualOffsetValues; import org.photonvision.vision.pipe.CVPipe.CVPipeResult; @@ -36,8 +32,6 @@ import org.photonvision.vision.target.TrackedTarget; /** Represents a pipeline for tracking retro-reflective targets. */ @SuppressWarnings({"DuplicatedCode"}) public class ReflectivePipeline extends CVPipeline { - private final RotateImagePipe rotateImagePipe = new RotateImagePipe(); - private final HSVPipe hsvPipe = new HSVPipe(); private final FindContoursPipe findContoursPipe = new FindContoursPipe(); private final SpeckleRejectPipe speckleRejectPipe = new SpeckleRejectPipe(); private final FilterContoursPipe filterContoursPipe = new FilterContoursPipe(); @@ -50,11 +44,15 @@ public class ReflectivePipeline extends CVPipeline hsvPipeResult; - Mat rawInputMat; - if (frame.image.getMat().channels() != 1) { - var rotateImageResult = rotateImagePipe.run(frame.image.getMat()); - sumPipeNanosElapsed += pipeProfileNanos[0] = rotateImageResult.nanosElapsed; - - rawInputMat = frame.image.getMat(); - - hsvPipeResult = hsvPipe.run(rawInputMat); - sumPipeNanosElapsed += hsvPipeResult.nanosElapsed; - pipeProfileNanos[1] = pipeProfileNanos[1] = hsvPipeResult.nanosElapsed; - } else { - // Try to copy the color frame. - long inputMatPtr = PicamJNI.grabFrame(true); - if (inputMatPtr != 0) { - // If we grabbed it (in color copy mode), make a new Mat of it - rawInputMat = new Mat(inputMatPtr); - } else { - // Otherwise, the input mat is frame we got from the camera - rawInputMat = frame.image.getMat(); - // // Otherwise, use a blank/empty mat as placeholder - // rawInputMat = new Mat(); - } - - // We can skip a few steps if the image is single channel because we've already done them on - // the GPU - hsvPipeResult = new CVPipeResult<>(); - hsvPipeResult.output = frame.image.getMat(); - hsvPipeResult.nanosElapsed = MathUtils.wpiNanoTime() - frame.timestampNanos; - - sumPipeNanosElapsed = pipeProfileNanos[1] = hsvPipeResult.nanosElapsed; - } - - CVPipeResult> findContoursResult = findContoursPipe.run(hsvPipeResult.output); + CVPipeResult> findContoursResult = + findContoursPipe.run(frame.processedImage.getMat()); sumPipeNanosElapsed += pipeProfileNanos[2] = findContoursResult.nanosElapsed; CVPipeResult> speckleRejectResult = @@ -226,11 +193,6 @@ public class ReflectivePipeline extends CVPipeline targets; - public final Frame outputFrame; - public final Frame inputFrame; + public final Frame inputAndOutputFrame; public CVPipelineResult( - double processingNanos, - double fps, - List targets, - Frame outputFrame, - Frame inputFrame) { + double processingNanos, double fps, List targets, Frame inputFrame) { this.processingNanos = processingNanos; this.fps = fps; this.targets = targets != null ? targets : Collections.emptyList(); - this.outputFrame = outputFrame; - this.inputFrame = inputFrame; + this.inputAndOutputFrame = inputFrame; } - public CVPipelineResult( - double processingNanos, double fps, List targets, Frame outputFrame) { - this(processingNanos, fps, targets, outputFrame, null); + public CVPipelineResult(double processingNanos, double fps, List targets) { + this(processingNanos, fps, targets, null); } public boolean hasTargets() { @@ -59,8 +52,7 @@ public class CVPipelineResult implements Releasable { for (TrackedTarget tt : targets) { tt.release(); } - outputFrame.release(); - if (inputFrame != null) inputFrame.release(); + if (inputAndOutputFrame != null) inputAndOutputFrame.release(); } /** @@ -68,6 +60,7 @@ public class CVPipelineResult implements Releasable { * the latency is relative to the time at which this method is called. Waiting to call this method * will change the latency this method returns. */ + @Deprecated public double getLatencyMillis() { var now = MathUtils.wpiNanoTime(); return MathUtils.nanosToMillis(now - imageCaptureTimestampNanos); diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java index 8d147bf0c..98fa1cba9 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java @@ -21,6 +21,7 @@ import edu.wpi.first.cscore.VideoException; import edu.wpi.first.math.util.Units; import io.javalin.websocket.WsContext; import java.util.*; +import java.util.function.BiConsumer; import org.photonvision.common.configuration.CameraConfiguration; import org.photonvision.common.configuration.ConfigManager; import org.photonvision.common.configuration.PhotonConfiguration; @@ -33,12 +34,11 @@ import org.photonvision.common.hardware.HardwareManager; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.common.util.SerializationUtils; -import org.photonvision.common.util.java.TriConsumer; import org.photonvision.vision.calibration.CameraCalibrationCoefficients; import org.photonvision.vision.camera.CameraQuirk; +import org.photonvision.vision.camera.LibcameraGpuSource; import org.photonvision.vision.camera.QuirkyCamera; import org.photonvision.vision.camera.USBCameraSource; -import org.photonvision.vision.camera.ZeroCopyPicamSource; import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.consumer.FileSaveFrameConsumer; import org.photonvision.vision.pipeline.AdvancedPipelineSettings; @@ -65,7 +65,7 @@ public class VisionModule { private final StreamRunnable streamRunnable; private final LinkedList resultConsumers = new LinkedList<>(); // Raw result consumers run before any drawing has been done by the OutputStreamPipeline - private final LinkedList>> streamResultConsumers = + private final LinkedList>> streamResultConsumers = new LinkedList<>(); private final NTDataPublisher ntConsumer; private final UIDataPublisher uiDataConsumer; @@ -93,7 +93,7 @@ public class VisionModule { // Find quirks for the current camera if (visionSource instanceof USBCameraSource) { cameraQuirks = ((USBCameraSource) visionSource).cameraQuirks; - } else if (visionSource instanceof ZeroCopyPicamSource) { + } else if (visionSource instanceof LibcameraGpuSource) { cameraQuirks = QuirkyCamera.ZeroCopyPiCamera; } else { cameraQuirks = QuirkyCamera.DefaultCamera; @@ -190,17 +190,29 @@ public class VisionModule { } private void recreateStreamResultConsumers() { - streamResultConsumers.add((in, out, tgts) -> inputFrameSaver.accept(in)); - streamResultConsumers.add((in, out, tgts) -> outputFrameSaver.accept(out)); - streamResultConsumers.add((in, out, tgts) -> inputVideoStreamer.accept(in)); - streamResultConsumers.add((in, out, tgts) -> outputVideoStreamer.accept(out)); + streamResultConsumers.add( + (frame, tgts) -> { + if (frame != null) inputFrameSaver.accept(frame.colorImage); + }); + streamResultConsumers.add( + (frame, tgts) -> { + if (frame != null) outputFrameSaver.accept(frame.processedImage); + }); + streamResultConsumers.add( + (frame, tgts) -> { + if (frame != null) inputVideoStreamer.accept(frame.colorImage); + }); + streamResultConsumers.add( + (frame, tgts) -> { + if (frame != null) outputVideoStreamer.accept(frame.processedImage); + }); } private class StreamRunnable extends Thread { private final OutputStreamPipeline outputStreamPipeline; private final Object frameLock = new Object(); - private Frame inputFrame, outputFrame; + private Frame latestFrame; private AdvancedPipelineSettings settings = new AdvancedPipelineSettings(); private List targets = new ArrayList<>(); @@ -211,42 +223,35 @@ public class VisionModule { } public void updateData( - Frame inputFrame, - Frame outputFrame, - AdvancedPipelineSettings settings, - List targets) { + Frame inputOutputFrame, AdvancedPipelineSettings settings, List targets) { synchronized (frameLock) { - if (shouldRun && this.inputFrame != null && this.outputFrame != null) { + if (shouldRun && this.latestFrame != null) { logger.trace("Fell behind; releasing last unused Mats"); - this.inputFrame.release(); - this.outputFrame.release(); + this.latestFrame.release(); } - this.inputFrame = inputFrame; - this.outputFrame = outputFrame; + this.latestFrame = inputOutputFrame; this.settings = settings; this.targets = targets; - shouldRun = - inputFrame != null - && !inputFrame.image.getMat().empty() - && outputFrame != null - && !outputFrame.image.getMat().empty(); + shouldRun = inputOutputFrame != null; + // && inputOutputFrame.colorImage != null + // && !inputOutputFrame.colorImage.getMat().empty() + // && inputOutputFrame.processedImage != null + // && !inputOutputFrame.processedImage.getMat().empty(); } } @Override public void run() { while (true) { - final Frame inputFrame, outputFrame; + final Frame m_frame; final AdvancedPipelineSettings settings; final List targets; final boolean shouldRun; synchronized (frameLock) { - inputFrame = this.inputFrame; - outputFrame = this.outputFrame; - this.inputFrame = null; - this.outputFrame = null; + m_frame = this.latestFrame; + this.latestFrame = null; settings = this.settings; targets = this.targets; @@ -256,17 +261,15 @@ public class VisionModule { } if (shouldRun) { try { - CVPipelineResult osr = - outputStreamPipeline.process(inputFrame, outputFrame, settings, targets); - consumeResults(inputFrame, osr.outputFrame, targets); + CVPipelineResult osr = outputStreamPipeline.process(m_frame, settings, targets); + consumeResults(m_frame, targets); } catch (Exception e) { // Never die logger.error("Exception while running stream runnable!", e); } try { - inputFrame.release(); - outputFrame.release(); + m_frame.release(); } catch (Exception e) { logger.error("Exception freeing frames", e); } @@ -496,7 +499,7 @@ public class VisionModule { internalMap.put("fps", videoModes.get(k).fps); internalMap.put( "pixelFormat", - ((videoModes.get(k) instanceof ZeroCopyPicamSource.FPSRatedVideoMode) + ((videoModes.get(k) instanceof LibcameraGpuSource.FPSRatedVideoMode) ? "kPicam" : videoModes.get(k).pixelFormat.toString()) .substring(1)); // Remove the k prefix @@ -546,16 +549,15 @@ public class VisionModule { consumePipelineResult(result); // Pipelines like DriverMode and Calibrate3dPipeline have null output frames - if (result.inputFrame != null + if (result.inputAndOutputFrame != null && (pipelineManager.getCurrentPipelineSettings() instanceof AdvancedPipelineSettings)) { streamRunnable.updateData( - result.inputFrame, - result.outputFrame, + result.inputAndOutputFrame, (AdvancedPipelineSettings) pipelineManager.getCurrentPipelineSettings(), result.targets); // The streamRunnable manages releasing in this case } else { - consumeResults(result.inputFrame, result.outputFrame, result.targets); + consumeResults(result.inputAndOutputFrame, result.targets); result.release(); // In this case we don't bother with a separate streaming thread and we release @@ -569,9 +571,9 @@ public class VisionModule { } /** Consume stream/target results, no rate limiting applied */ - private void consumeResults(Frame inputFrame, Frame outputFrame, List targets) { + private void consumeResults(Frame frame, List targets) { for (var c : streamResultConsumers) { - c.accept(inputFrame, outputFrame, targets); + c.accept(frame, targets); } } diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java index f4ba5d6e7..60acc0d39 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java @@ -22,8 +22,9 @@ import java.util.function.Supplier; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.vision.camera.QuirkyCamera; -import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.FrameProvider; +import org.photonvision.vision.pipe.impl.HSVPipe; +import org.photonvision.vision.pipeline.AdvancedPipelineSettings; import org.photonvision.vision.pipeline.CVPipeline; import org.photonvision.vision.pipeline.result.CVPipelineResult; @@ -32,7 +33,7 @@ import org.photonvision.vision.pipeline.result.CVPipelineResult; public class VisionRunner { private final Logger logger; private final Thread visionProcessThread; - private final Supplier frameSupplier; + private final FrameProvider frameSupplier; private final Supplier pipelineSupplier; private final Consumer pipelineResultConsumer; private final QuirkyCamera cameraQuirks; @@ -69,8 +70,29 @@ public class VisionRunner { private void update() { while (!Thread.interrupted()) { var pipeline = pipelineSupplier.get(); + + // Tell our camera implementation here what kind of pre-processing we need it to be doing + // (pipeline-dependent). I kinda hate how much leak this has... + // TODO would a callback object be a better fit? + var wantedProcessType = pipeline.getThresholdType(); + frameSupplier.requestFrameThresholdType(wantedProcessType); + var settings = pipeline.getSettings(); + if (settings instanceof AdvancedPipelineSettings) { + var advanced = (AdvancedPipelineSettings) settings; + var hsvParams = + new HSVPipe.HSVParams( + advanced.hsvHue, advanced.hsvSaturation, advanced.hsvValue, advanced.hueInverted); + // TODO who should deal with preventing this from happening _every single loop_? + frameSupplier.requestHsvSettings(hsvParams); + } + frameSupplier.requestFrameRotation(settings.inputImageRotationMode); + frameSupplier.requestFrameCopies(settings.inputShouldShow, settings.outputShouldShow); + + // Grab the new camera frame var frame = frameSupplier.get(); + // There's no guarantee the processing type change will occur this tick, so pipelines should + // check themselves try { var pipelineResult = pipeline.run(frame, cameraQuirks); pipelineResultConsumer.accept(pipelineResult); @@ -78,6 +100,7 @@ public class VisionRunner { logger.error("Exception on loop " + loopCount); ex.printStackTrace(); } + loopCount++; } } diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java index 35fb48b4b..81f216922 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java @@ -27,14 +27,15 @@ import org.photonvision.common.configuration.CameraConfiguration; import org.photonvision.common.configuration.ConfigManager; import org.photonvision.common.dataflow.DataChangeService; import org.photonvision.common.dataflow.events.OutgoingUIEvent; +import org.photonvision.common.hardware.Platform; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.common.util.TimedTaskManager; -import org.photonvision.raspi.PicamJNI; +import org.photonvision.raspi.LibCameraJNI; import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.camera.CameraType; +import org.photonvision.vision.camera.LibcameraGpuSource; import org.photonvision.vision.camera.USBCameraSource; -import org.photonvision.vision.camera.ZeroCopyPicamSource; public class VisionSourceManager { private static final Logger logger = new Logger(VisionSourceManager.class, LogGroup.Camera); @@ -252,8 +253,14 @@ public class VisionSourceManager { logger.info("Creating a new camera config for camera " + uniqueName); + // HACK -- for picams, we want to use the camera model + String nickname = uniqueName; + if (isCsiCamera(info)) { + nickname = LibCameraJNI.getSensorModel().toString(); + } + CameraConfiguration configuration = - new CameraConfiguration(baseName, uniqueName, uniqueName, info.path); + new CameraConfiguration(baseName, uniqueName, nickname, info.path, info.otherPaths); cameraConfigurations.add(configuration); } @@ -261,6 +268,11 @@ public class VisionSourceManager { return cameraConfigurations; } + private boolean isCsiCamera(UsbCameraInfo configuration) { + return (Arrays.stream(configuration.otherPaths).anyMatch(it -> it.contains("csi-video")) + || cameraNameToBaseName(configuration.name).equals("unicam")); + } + private CameraConfiguration mergeInfoIntoConfig(CameraConfiguration cfg, UsbCameraInfo info) { if (!cfg.path.equals(info.path)) { logger.debug("Updating path config from " + cfg.path + " to " + info.path); @@ -307,20 +319,23 @@ public class VisionSourceManager { List camConfigs) { var cameraSources = new ArrayList(); for (var configuration : camConfigs) { - if (configuration.baseName.startsWith("mmal service") && PicamJNI.isSupported()) { - configuration.cameraType = CameraType.ZeroCopyPicam; - var piCamSrc = new ZeroCopyPicamSource(configuration); + System.out.println("Creating VisionSource for " + configuration); + // Picams should have csi-video in the path + boolean is_picam = + (Arrays.stream(configuration.otherPaths).anyMatch(it -> it.contains("csi-video")) + || configuration.baseName.equals("unicam")); + boolean is_pi = Platform.isRaspberryPi(); + if (is_picam && is_pi) { configuration.cameraType = CameraType.ZeroCopyPicam; + var piCamSrc = new LibcameraGpuSource(configuration); cameraSources.add(piCamSrc); - continue; - } - - var newCam = new USBCameraSource(configuration); - - if (!newCam.cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken) - && !newCam.getSettables().videoModes.isEmpty()) { - cameraSources.add(newCam); + } else { + var newCam = new USBCameraSource(configuration); + if (!newCam.cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken) + && !newCam.getSettables().videoModes.isEmpty()) { + cameraSources.add(newCam); + } } } return cameraSources; diff --git a/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java b/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java index de72a8cf3..d4eef5e4f 100644 --- a/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java +++ b/photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java @@ -25,10 +25,10 @@ import org.opencv.core.MatOfByte; import org.opencv.core.MatOfInt; import org.opencv.imgcodecs.Imgcodecs; import org.photonvision.common.util.math.MathUtils; -import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.consumer.MJPGFrameConsumer; +import org.photonvision.vision.opencv.CVMat; -public class SocketVideoStream implements Consumer { +public class SocketVideoStream implements Consumer { int portID = 0; // Align with cscore's port for unique identification of stream MatOfByte jpegBytes = null; @@ -55,7 +55,7 @@ public class SocketVideoStream implements Consumer { } @Override - public void accept(Frame frame) { + public void accept(CVMat image) { if (userCount > 0) { if (jpegBytesLock .tryLock()) { // we assume frames are coming in frequently. Just skip this frame if we're @@ -63,12 +63,12 @@ public class SocketVideoStream implements Consumer { try { // Does a single-shot frame recieve and convert to JPEG for efficency // Will not capture/convert again until convertNextFrame() is called - if (frame != null && !frame.image.getMat().empty() && jpegBytes == null) { + if (image != null && !image.getMat().empty() && jpegBytes == null) { frameWasConsumed = false; jpegBytes = new MatOfByte(); Imgcodecs.imencode( ".jpg", - frame.image.getMat(), + image.getMat(), jpegBytes, new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 75)); } @@ -81,7 +81,7 @@ public class SocketVideoStream implements Consumer { // Send the frame in an FPS-limited fashion var now = MathUtils.wpiNanoTime(); if (now > nextFrameSendTime) { - oldSchoolServer.accept(frame); + oldSchoolServer.accept(image); nextFrameSendTime = now + minFramePeriodNanos; } } diff --git a/photon-core/src/test/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProviderTest.java b/photon-core/src/test/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProviderTest.java deleted file mode 100644 index a62285438..000000000 --- a/photon-core/src/test/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProviderTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) Photon Vision. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.photonvision.vision.frame.provider; - -import java.io.IOException; -import org.junit.jupiter.api.Test; -import org.opencv.imgcodecs.Imgcodecs; -import org.photonvision.common.configuration.CameraConfiguration; -import org.photonvision.common.util.TestUtils; -import org.photonvision.raspi.PicamJNI; -import org.photonvision.vision.camera.ZeroCopyPicamSource; - -public class AcceleratedPicamFrameProviderTest { - @Test - public void testGrabFrame() throws IOException { - PicamJNI.forceLoad(); - if (!PicamJNI.isSupported()) return; - - TestUtils.loadLibraries(); - - var frameProvider = - new AcceleratedPicamFrameProvider( - new ZeroCopyPicamSource.PicamSettables(new CameraConfiguration("f", "f", "f", "f"))); - - long lastTime = System.currentTimeMillis(); - for (int i = 0; i < 10; i++) { - var frame = frameProvider.get(); - System.out.println(frame.image.getMat().get(0, 0)[0]); - - long time = System.currentTimeMillis(); - System.out.println("dt (ms): " + (time - lastTime)); - lastTime = time; - } - var mat = frameProvider.get().image.getMat(); - Imgcodecs.imwrite("out.png", mat); - } -} diff --git a/photon-core/src/test/java/org/photonvision/vision/frame/provider/FileFrameProviderTest.java b/photon-core/src/test/java/org/photonvision/vision/frame/provider/FileFrameProviderTest.java index cad06f936..05cddeb9d 100644 --- a/photon-core/src/test/java/org/photonvision/vision/frame/provider/FileFrameProviderTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/frame/provider/FileFrameProviderTest.java @@ -48,14 +48,14 @@ public class FileFrameProviderTest { Frame goodFrame = goodFrameProvider.get(); - int goodFrameCols = goodFrame.image.getMat().cols(); - int goodFrameRows = goodFrame.image.getMat().rows(); + int goodFrameCols = goodFrame.colorImage.getMat().cols(); + int goodFrameRows = goodFrame.colorImage.getMat().rows(); // 2019 Images are at 320x240 assertEquals(320, goodFrameCols); assertEquals(240, goodFrameRows); - TestUtils.showImage(goodFrame.image.getMat(), "2019"); + TestUtils.showImage(goodFrame.colorImage.getMat(), "2019"); var badFilePath = Paths.get("bad.jpg"); // this file does not exist @@ -81,14 +81,14 @@ public class FileFrameProviderTest { Frame goodFrame = goodFrameProvider.get(); - int goodFrameCols = goodFrame.image.getMat().cols(); - int goodFrameRows = goodFrame.image.getMat().rows(); + int goodFrameCols = goodFrame.colorImage.getMat().cols(); + int goodFrameRows = goodFrame.colorImage.getMat().rows(); // 2020 Images are at 640x480 assertEquals(640, goodFrameCols); assertEquals(480, goodFrameRows); - TestUtils.showImage(goodFrame.image.getMat(), "2020"); + TestUtils.showImage(goodFrame.colorImage.getMat(), "2020"); var badFilePath = Paths.get("bad.jpg"); // this file does not exist diff --git a/photon-core/src/test/java/org/photonvision/vision/frame/provider/LibcameraTest.java b/photon-core/src/test/java/org/photonvision/vision/frame/provider/LibcameraTest.java new file mode 100644 index 000000000..e72d84b03 --- /dev/null +++ b/photon-core/src/test/java/org/photonvision/vision/frame/provider/LibcameraTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.frame.provider; + +import org.junit.jupiter.api.Test; +// import org.photonvision.raspi.LibCameraJNI; + +public class LibcameraTest { + @Test + public void testBasic() { + // System.load("/home/pi/photon-libcamera-gl-driver/build/libphotonlibcamera.so"); + // LibCameraJNI.createCamera(1920, 1080, 60); + // try { + // Thread.sleep(1000); + // } catch (InterruptedException e) { + // } + // LibCameraJNI.startCamera(); + // try { + // Thread.sleep(5000); + // } catch (InterruptedException e) { + // } + } +} diff --git a/photon-core/src/test/java/org/photonvision/vision/pipeline/AprilTagTest.java b/photon-core/src/test/java/org/photonvision/vision/pipeline/AprilTagTest.java index 7770d42d8..9fede51cd 100644 --- a/photon-core/src/test/java/org/photonvision/vision/pipeline/AprilTagTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/pipeline/AprilTagTest.java @@ -52,6 +52,7 @@ public class AprilTagTest { TestUtils.getApriltagImagePath(TestUtils.ApriltagTestImages.kTag1_640_480, false), TestUtils.WPI2020Image.FOV, TestUtils.get2020LifeCamCoeffs(false)); + frameProvider.requestFrameThresholdType(pipeline.getThresholdType()); CVPipelineResult pipelineResult; try { @@ -65,12 +66,10 @@ public class AprilTagTest { // Draw on input var outputPipe = new OutputStreamPipeline(); outputPipe.process( - pipelineResult.inputFrame, - pipelineResult.outputFrame, - pipeline.getSettings(), - pipelineResult.targets); + pipelineResult.inputAndOutputFrame, pipeline.getSettings(), pipelineResult.targets); - TestUtils.showImage(pipelineResult.inputFrame.image.getMat(), "Pipeline output", 999999); + TestUtils.showImage( + pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999); // these numbers are not *accurate*, but they are known and expected var pose = pipelineResult.targets.get(0).getBestCameraToTarget3d(); diff --git a/photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java b/photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java index e6886c8a0..6f54d2303 100644 --- a/photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java @@ -38,6 +38,7 @@ import org.photonvision.vision.calibration.CameraCalibrationCoefficients; import org.photonvision.vision.camera.QuirkyCamera; import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.FrameStaticProperties; +import org.photonvision.vision.frame.FrameThresholdType; import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.pipe.impl.Calibrate3dPipe; import org.photonvision.vision.pipe.impl.FindBoardCornersPipe; @@ -103,9 +104,11 @@ public class Calibrate3dPipeTest { var frame = new Frame( new CVMat(Imgcodecs.imread(file.getAbsolutePath())), + new CVMat(), + FrameThresholdType.NONE, new FrameStaticProperties(640, 480, 60, null)); var output = calibration3dPipeline.run(frame, QuirkyCamera.DefaultCamera); - // TestUtils.showImage(output.outputFrame.image.getMat()); + // TestUtils.showImage(output.inputAndOutputFrame.processedImage.getMat()); output.release(); frame.release(); } @@ -119,6 +122,8 @@ public class Calibrate3dPipeTest { var frame = new Frame( new CVMat(Imgcodecs.imread(directoryListing[0].getAbsolutePath())), + new CVMat(), + FrameThresholdType.NONE, new FrameStaticProperties(640, 480, 60, null)); calibration3dPipeline.run(frame, QuirkyCamera.DefaultCamera).release(); frame.release(); @@ -266,10 +271,13 @@ public class Calibrate3dPipeTest { var frame = new Frame( new CVMat(Imgcodecs.imread(file.getAbsolutePath())), + new CVMat(), + FrameThresholdType.NONE, new FrameStaticProperties((int) imgRes.width, (int) imgRes.height, 67, null)); var output = calibration3dPipeline.run(frame, QuirkyCamera.DefaultCamera); - // TestUtils.showImage(output.outputFrame.image.getMat(), file.getName(), 1); + // TestUtils.showImage(output.inputAndOutputFrame.processedImage.getMat(), file.getName(), + // 1); output.release(); frame.release(); } diff --git a/photon-core/src/test/java/org/photonvision/vision/pipeline/CirclePNPTest.java b/photon-core/src/test/java/org/photonvision/vision/pipeline/CirclePNPTest.java index ed67e2dcf..14a51b230 100644 --- a/photon-core/src/test/java/org/photonvision/vision/pipeline/CirclePNPTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/pipeline/CirclePNPTest.java @@ -111,11 +111,13 @@ public class CirclePNPTest { TestUtils.getPowercellImagePath(TestUtils.PowercellTestImages.kPowercell_test_6, false), TestUtils.WPI2020Image.FOV, TestUtils.get2020LifeCamCoeffs(true)); + frameProvider.requestFrameThresholdType(pipeline.getThresholdType()); CVPipelineResult pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera); printTestResults(pipelineResult); - TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output", 999999); + TestUtils.showImage( + pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999); } private static void continuouslyRunPipeline(Frame frame, ReflectivePipelineSettings settings) { diff --git a/photon-core/src/test/java/org/photonvision/vision/pipeline/ColoredShapePipelineTest.java b/photon-core/src/test/java/org/photonvision/vision/pipeline/ColoredShapePipelineTest.java index 10fa3c2cb..b89618562 100644 --- a/photon-core/src/test/java/org/photonvision/vision/pipeline/ColoredShapePipelineTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/pipeline/ColoredShapePipelineTest.java @@ -33,7 +33,8 @@ public class ColoredShapePipelineTest { pipeline.settings = settings; CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera); TestUtils.showImage( - colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Triangle."); + colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(), + "Pipeline output: Triangle."); printTestResults(colouredShapePipelineResult); } @@ -43,7 +44,8 @@ public class ColoredShapePipelineTest { pipeline.settings = settings; CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera); TestUtils.showImage( - colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Quadrilateral."); + colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(), + "Pipeline output: Quadrilateral."); printTestResults(colouredShapePipelineResult); } @@ -53,7 +55,8 @@ public class ColoredShapePipelineTest { pipeline.settings = settings; CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera); TestUtils.showImage( - colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Custom."); + colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(), + "Pipeline output: Custom."); printTestResults(colouredShapePipelineResult); } @@ -64,7 +67,8 @@ public class ColoredShapePipelineTest { pipeline.settings = settings; CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera); TestUtils.showImage( - colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Circle."); + colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(), + "Pipeline output: Circle."); printTestResults(colouredShapePipelineResult); } diff --git a/photon-core/src/test/java/org/photonvision/vision/pipeline/ReflectivePipelineTest.java b/photon-core/src/test/java/org/photonvision/vision/pipeline/ReflectivePipelineTest.java index 0a8e74268..0817cf37d 100644 --- a/photon-core/src/test/java/org/photonvision/vision/pipeline/ReflectivePipelineTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/pipeline/ReflectivePipelineTest.java @@ -26,6 +26,7 @@ import org.photonvision.vision.frame.provider.FileFrameProvider; import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.opencv.ContourGroupingMode; import org.photonvision.vision.opencv.ContourIntersectionDirection; +import org.photonvision.vision.pipe.impl.HSVPipe; import org.photonvision.vision.pipeline.result.CVPipelineResult; public class ReflectivePipelineTest { @@ -45,8 +46,16 @@ public class ReflectivePipelineTest { new FileFrameProvider( TestUtils.getWPIImagePath(TestUtils.WPI2019Image.kCargoStraightDark72in_HighRes, false), TestUtils.WPI2019Image.FOV); + frameProvider.requestFrameThresholdType(pipeline.getThresholdType()); + var hsvParams = + new HSVPipe.HSVParams( + pipeline.getSettings().hsvHue, + pipeline.getSettings().hsvSaturation, + pipeline.getSettings().hsvValue, + pipeline.getSettings().hueInverted); + frameProvider.requestHsvSettings(hsvParams); - TestUtils.showImage(frameProvider.get().image.getMat(), "Pipeline input", 1); + TestUtils.showImage(frameProvider.get().colorImage.getMat(), "Pipeline input", 1); CVPipelineResult pipelineResult; @@ -56,7 +65,7 @@ public class ReflectivePipelineTest { Assertions.assertTrue(pipelineResult.hasTargets()); Assertions.assertEquals(2, pipelineResult.targets.size(), "Target count wrong!"); - TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output"); + TestUtils.showImage(pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output"); } @Test @@ -77,7 +86,8 @@ public class ReflectivePipelineTest { CVPipelineResult pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera); printTestResults(pipelineResult); - TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output"); + TestUtils.showImage( + pipelineResult.inputAndOutputFrame.processedImage.getMat(), "Pipeline output"); } private static void continuouslyRunPipeline(Frame frame, ReflectivePipelineSettings settings) { diff --git a/photon-core/src/test/java/org/photonvision/vision/pipeline/SolvePNPTest.java b/photon-core/src/test/java/org/photonvision/vision/pipeline/SolvePNPTest.java index dbf1df8d2..02852308b 100644 --- a/photon-core/src/test/java/org/photonvision/vision/pipeline/SolvePNPTest.java +++ b/photon-core/src/test/java/org/photonvision/vision/pipeline/SolvePNPTest.java @@ -34,6 +34,7 @@ import org.photonvision.vision.frame.provider.FileFrameProvider; import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.opencv.ContourGroupingMode; import org.photonvision.vision.opencv.ContourIntersectionDirection; +import org.photonvision.vision.pipe.impl.HSVPipe; import org.photonvision.vision.pipeline.result.CVPipelineResult; import org.photonvision.vision.target.TargetModel; import org.photonvision.vision.target.TrackedTarget; @@ -105,6 +106,15 @@ public class SolvePNPTest { TestUtils.WPI2019Image.FOV, TestUtils.get2019LifeCamCoeffs(false)); + frameProvider.requestFrameThresholdType(pipeline.getThresholdType()); + var hsvParams = + new HSVPipe.HSVParams( + pipeline.getSettings().hsvHue, + pipeline.getSettings().hsvSaturation, + pipeline.getSettings().hsvValue, + pipeline.getSettings().hueInverted); + frameProvider.requestHsvSettings(hsvParams); + CVPipelineResult pipelineResult; pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera); @@ -125,7 +135,8 @@ public class SolvePNPTest { Assertions.assertEquals( 1, new Translation3d(0, 0, 1).rotateBy(pose.getRotation()).getZ(), 0.05); - TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output", 999999); + TestUtils.showImage( + pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999); } @Test @@ -147,16 +158,22 @@ public class SolvePNPTest { TestUtils.WPI2020Image.FOV, TestUtils.get2020LifeCamCoeffs(false)); + frameProvider.requestFrameThresholdType(pipeline.getThresholdType()); + var hsvParams = + new HSVPipe.HSVParams( + pipeline.getSettings().hsvHue, + pipeline.getSettings().hsvSaturation, + pipeline.getSettings().hsvValue, + pipeline.getSettings().hueInverted); + frameProvider.requestHsvSettings(hsvParams); + CVPipelineResult pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera); printTestResults(pipelineResult); // Draw on input var outputPipe = new OutputStreamPipeline(); outputPipe.process( - pipelineResult.inputFrame, - pipelineResult.outputFrame, - pipeline.getSettings(), - pipelineResult.targets); + pipelineResult.inputAndOutputFrame, pipeline.getSettings(), pipelineResult.targets); // these numbers are not *accurate*, but they are known and expected var pose = pipelineResult.targets.get(0).getBestCameraToTarget3d(); @@ -165,7 +182,8 @@ public class SolvePNPTest { // Z rotation should be mostly facing us Assertions.assertEquals(Units.degreesToRadians(-140), pose.getRotation().getZ(), 1); - TestUtils.showImage(pipelineResult.inputFrame.image.getMat(), "Pipeline output", 999999); + TestUtils.showImage( + pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999); } private static void continuouslyRunPipeline(Frame frame, ReflectivePipelineSettings settings) { diff --git a/photon-server/src/main/java/org/photonvision/Main.java b/photon-server/src/main/java/org/photonvision/Main.java index 76aa90a71..e41031c65 100644 --- a/photon-server/src/main/java/org/photonvision/Main.java +++ b/photon-server/src/main/java/org/photonvision/Main.java @@ -37,6 +37,7 @@ import org.photonvision.common.logging.Logger; import org.photonvision.common.networking.NetworkManager; import org.photonvision.common.util.TestUtils; import org.photonvision.common.util.numbers.IntegerCouple; +import org.photonvision.raspi.LibCameraJNI; import org.photonvision.server.Server; import org.photonvision.vision.camera.FileVisionSource; import org.photonvision.vision.opencv.CVMat; @@ -293,11 +294,11 @@ public class Main { logger.error("Failed to load native libraries!", e); } - // try { - // PicamJNI.forceLoad(); - // } catch (IOException e) { - // logger.error("Failed to load Picam JNI!", e); - // } + try { + LibCameraJNI.forceLoad(); + } catch (IOException e) { + logger.error("Failed to load native libraries!", e); + } try { if (!handleArgs(args)) { diff --git a/photon-server/src/main/resources/nativelibraries/libphotonlibcamera.so b/photon-server/src/main/resources/nativelibraries/libphotonlibcamera.so new file mode 100755 index 000000000..44d8bd133 Binary files /dev/null and b/photon-server/src/main/resources/nativelibraries/libphotonlibcamera.so differ diff --git a/scripts/generatePiImage.sh b/scripts/generatePiImage.sh index b85486a46..02590cc7c 100755 --- a/scripts/generatePiImage.sh +++ b/scripts/generatePiImage.sh @@ -16,8 +16,24 @@ LOOP=$(sudo losetup --show -fP "${IMAGE_FILE}") sudo mount ${LOOP}p2 $TMP pushd . cd $TMP/opt/photonvision -ls sudo cp $NEW_JAR photonvision.jar + +cd $TMP/etc/systemd/system/multi-user.target.wants +sudo bash -c "printf \ +\"[Unit] +Description=Service that runs PhotonVision + +[Service] +WorkingDirectory=/opt/photonvision +ExecStart=/usr/bin/java -Xmx512m -jar /opt/photonvision/photonvision.jar +ExecStop=/bin/systemctl kill photonvision +Type=simple +Restart=on-failure +RestartSec=1 + +[Install] +WantedBy=multi-user.target\" > photonvision.service" + popd sudo umount ${TMP} sudo rmdir ${TMP} diff --git a/scripts/install.sh b/scripts/install.sh index d386b4e1a..e988eaead 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -68,7 +68,7 @@ Nice=-10 # look up the right values for your CPU # AllowCPUs=4-7 -ExecStart=/usr/bin/java -jar /opt/photonvision/photonvision.jar -Xmx512m +ExecStart=/usr/bin/java -Xmx512m -jar /opt/photonvision/photonvision.jar ExecStop=/bin/systemctl kill photonvision Type=simple Restart=on-failure