Team2890/PhotonVision
4
0
Fork 0
mirror of https://github.com/PhotonVision/photonvision synced 2026-06-19 00:41:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace MMAL driver with Libcamera (#491)

Browse Source 
Co-authored-by: Chris Gerth <gerth2@users.noreply.github.com>
This commit is contained in:
Matt
2022-12-28 11:21:41 -08:00
committed by GitHub
parent 4088a394f3
commit 7ff630dc44
53 changed files with 1261 additions and 934 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/main.yml vendored
View File

@@ -375,7 +375,7 @@ jobs:
./gradlew photon-server:shadowJar --max-workers 2
if: ${{ (matrix.arch-override == 'none') }}
# The image will only pull the Pi JAR in
# The image will only pull the Pi32 JAR in
- name: Generate image
if: ${{ github.event_name != 'pull_request' && (matrix.artifact-name) == 'LinuxArm32' }}
run: |

3
build.gradle
View File

@@ -36,9 +36,10 @@ ext {
jniPlatforms = ['linuxarm32']
} else if(project.hasProperty('winonly')) {
jniPlatforms = ['windowsx86-64']
} else if(project.hasProperty('aarch64only')) {
jniPlatforms = ['linuxaarch64bionic']
} else {
jniPlatforms = ['linuxarm64', 'linuxarm32', 'linuxx86-64', 'osxuniversal', 'windowsx86-64']
}
println("Building for archs " + jniPlatforms)

25
photon-client/package-lock.json generated
View File

@@ -5,7 +5,6 @@
"requires": true,
"packages": {
"": {
"name": "photon-client",
"version": "3.0.0",
"dependencies": {
"@femessage/log-viewer": "^1.4.2",
@@ -19,7 +18,8 @@
"three-full": "^28.0.2",
"vue": "^2.6.12",
"vue-axios": "^2.1.5",
"vue-native-websocket": "git+https://github.com/PhotonVision/vue-native-websocket.git#5189f29", "vue-router": "^3.4.3",
"vue-native-websocket": "git+https://git@github.com/PhotonVision/vue-native-websocket.git#5189f29",
"vue-router": "^3.4.3",
"vuetify": "^2.3.10",
"vuex": "^3.5.1"
},
@@ -2682,6 +2682,7 @@
"thread-loader": "^2.1.3",
"url-loader": "^2.2.0",
"vue-loader": "^15.9.2",
"vue-loader-v16": "npm:vue-loader@^16.0.0-beta.3",
"vue-style-loader": "^4.1.2",
"webpack": "^4.0.0",
"webpack-bundle-analyzer": "^3.8.0",
@@ -3115,6 +3116,7 @@
"merge-source-map": "^1.1.0",
"postcss": "^7.0.14",
"postcss-selector-parser": "^6.0.2",
"prettier": "^1.18.2",
"source-map": "~0.6.1",
"vue-template-es2015-compiler": "^1.9.0"
},
@@ -4701,6 +4703,7 @@
"dependencies": {
"anymatch": "~3.1.1",
"braces": "~3.0.2",
"fsevents": "~2.1.2",
"glob-parent": "~5.1.0",
"is-binary-path": "~2.1.0",
"is-glob": "~4.0.1",
@@ -9164,6 +9167,9 @@
"version": "4.0.0",
"dev": true,
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.1.6"
},
"optionalDependencies": {
"graceful-fs": "^4.1.6"
}
@@ -9175,7 +9181,11 @@
"@babel/runtime": "^7.14.0",
"atob": "^2.1.2",
"btoa": "^1.2.1",
"fflate": "^0.4.8"
"canvg": "^3.0.6",
"core-js": "^3.6.0",
"dompurify": "^2.2.0",
"fflate": "^0.4.8",
"html2canvas": "^1.0.0-rc.5"
},
"optionalDependencies": {
"canvg": "^3.0.6",
@@ -14190,8 +14200,10 @@
"dev": true,
"license": "MIT",
"dependencies": {
"chokidar": "^3.4.0",
"graceful-fs": "^4.1.2",
"neo-async": "^2.5.0"
"neo-async": "^2.5.0",
"watchpack-chokidar2": "^2.0.0"
},
"optionalDependencies": {
"chokidar": "^3.4.0",
@@ -14300,6 +14312,7 @@
"anymatch": "^2.0.0",
"async-each": "^1.0.1",
"braces": "^2.3.2",
"fsevents": "^1.2.7",
"glob-parent": "^3.1.0",
"inherits": "^2.0.3",
"is-binary-path": "^1.0.0",
@@ -14540,6 +14553,7 @@
"anymatch": "^2.0.0",
"async-each": "^1.0.1",
"braces": "^2.3.2",
"fsevents": "^1.2.7",
"glob-parent": "^3.1.0",
"inherits": "^2.0.3",
"is-binary-path": "^1.0.0",
@@ -25270,7 +25284,8 @@
},
"vue-native-websocket": {
"version": "git+ssh://git@github.com/PhotonVision/vue-native-websocket.git#7a327918e03b215b6899b0d648c5130ece1fa912",
"from": "vue-native-websocket@git+https://github.com/PhotonVision/vue-native-websocket.git#7a32791" },
"from": "vue-native-websocket@git+https://git@github.com/PhotonVision/vue-native-websocket.git#5189f29"
},
"vue-router": {
"version": "3.4.3"
},

20
photon-client/src/views/CamerasView.vue
View File

@@ -218,6 +218,17 @@
tooltip="Enables or Disables camera automatic adjustment for current lighting conditions"
@input="e => handlePipelineUpdate('cameraAutoExposure', e)"
/>
<CVslider
v-if="cameraGain >= 0"
v-model="cameraGain"
name="Camera Gain"
min="0"
max="100"
tooltip="Controls camera gain, similar to brightness"
:slider-cols="largeBox"
@input="handlePipelineData('cameraGain')"
@rollback="e => rollback('cameraGain', e)"
/>
<CVslider
v-if="$store.getters.currentPipelineSettings.cameraRedGain !== -1"
v-model="$store.getters.currentPipelineSettings.cameraRedGain"
@@ -410,6 +421,15 @@ export default {
}
},
cameraGain: {
get() {
return parseInt(this.$store.getters.currentPipelineSettings.cameraGain)
},
set(val) {
this.$store.commit("mutatePipeline", {"cameraGain": parseInt(val)});
}
},
// Makes sure there's only one entry per resolution
filteredResolutionList: {
get() {

6
photon-client/src/views/PipelineView.vue
View File

@@ -34,9 +34,9 @@
:text-color="fpsTooLow ? 'white' : 'grey'"
>
<span class="pr-1">Processing @ {{ Math.round($store.state.pipelineResults.fps) }}&nbsp;FPS &ndash;</span>
<span v-if="!fpsTooLow">{{ Math.min(Math.round($store.state.pipelineResults.latency), 9999) }} ms latency</span>
<span v-else-if="!$store.getters.currentPipelineSettings.inputShouldShow">HSV thresholds are too broad; narrow them for better performance</span>
<span v-else>stop viewing the raw stream for better performance</span>
<span v-if="fpsTooLow && !$store.getters.currentPipelineSettings.inputShouldShow && $store.getters.pipelineType == 2">HSV thresholds are too broad; narrow them for better performance</span>
<span v-else-if="$fpsTooLow && getters.currentCameraSettings.inputShouldShow">stop viewing the raw stream for better performance</span>
<span v-else>{{ Math.min(Math.round($store.state.pipelineResults.latency), 9999) }} ms latency</span>
</v-chip>
<v-switch
v-model="driverMode"

38
photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java
View File

@@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
@@ -45,6 +46,8 @@ public class CameraConfiguration {
/** Can be either path (ex /dev/videoX) or index (ex 1). */
public String path = "";
@JsonIgnore public String[] otherPaths = {};
public CameraType cameraType = CameraType.UsbCamera;
public double FOV = 70;
public final List<CameraCalibrationCoefficients> calibrations;
@@ -59,15 +62,17 @@ public class CameraConfiguration {
public DriverModePipelineSettings driveModeSettings = new DriverModePipelineSettings();
public CameraConfiguration(String baseName, String path) {
this(baseName, baseName, baseName, path);
this(baseName, baseName, baseName, path, new String[0]);
}
public CameraConfiguration(String baseName, String uniqueName, String nickname, String path) {
public CameraConfiguration(
String baseName, String uniqueName, String nickname, String path, String[] alternates) {
this.baseName = baseName;
this.uniqueName = uniqueName;
this.nickname = nickname;
this.path = path;
this.calibrations = new ArrayList<>();
this.otherPaths = alternates;
logger.debug(
"Creating USB camera configuration for "
@@ -145,4 +150,33 @@ public class CameraConfiguration {
.ifPresent(calibrations::remove);
calibrations.add(calibration);
}
@Override
public String toString() {
return "CameraConfiguration [baseName="
+ baseName
+ ", uniqueName="
+ uniqueName
+ ", nickname="
+ nickname
+ ", path="
+ path
+ ", otherPaths="
+ Arrays.toString(otherPaths)
+ ", cameraType="
+ cameraType
+ ", FOV="
+ FOV
+ ", calibrations="
+ calibrations
+ ", currentPipelineIndex="
+ currentPipelineIndex
+ ", streamIndex="
+ streamIndex
+ ", pipelineSettings="
+ pipelineSettings
+ ", driveModeSettings="
+ driveModeSettings
+ "]";
}
}

6
photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java
View File

@@ -25,7 +25,7 @@ import java.util.stream.Collectors;
import org.photonvision.PhotonVersion;
import org.photonvision.common.hardware.Platform;
import org.photonvision.common.util.SerializationUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.raspi.LibCameraJNI;
import org.photonvision.vision.processes.VisionModule;
import org.photonvision.vision.processes.VisionModuleManager;
import org.photonvision.vision.processes.VisionSource;
@@ -110,8 +110,8 @@ public class PhotonConfiguration {
generalSubmap.put("version", PhotonVersion.versionString);
generalSubmap.put(
"gpuAcceleration",
PicamJNI.isSupported()
? "Zerocopy MMAL on " + PicamJNI.getSensorModel().getFriendlyName()
LibCameraJNI.isSupported()
? "Zerocopy Libcamera on " + LibCameraJNI.getSensorModel().getFriendlyName()
: ""); // TODO add support for other types of GPU accel
generalSubmap.put("hardwareModel", hardwareConfig.deviceName);
generalSubmap.put("hardwarePlatform", Platform.getPlatformName());

2
photon-core/src/main/java/org/photonvision/common/hardware/PiVersion.java
View File

@@ -34,7 +34,7 @@ public enum PiVersion {
private static final ShellExec shell = new ShellExec(true, false);
private static final PiVersion currentPiVersion = calcPiVersion();
PiVersion(String s) {
private PiVersion(String s) {
this.identifier = s.toLowerCase();
}

1
photon-core/src/main/java/org/photonvision/common/util/TestUtils.java
View File

@@ -330,6 +330,7 @@ public class TestUtils {
private static int DefaultTimeoutMillis = 5000;
public static void showImage(Mat frame, String title, int timeoutMs) {
if (frame.empty()) return;
try {
HighGui.imshow(title, frame);
HighGui.waitKey(timeoutMs);

191
photon-core/src/main/java/org/photonvision/raspi/LibCameraJNI.java Normal file
View File

@@ -0,0 +1,191 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.raspi;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
public class LibCameraJNI {
private static boolean libraryLoaded = false;
private static Logger logger = new Logger(LibCameraJNI.class, LogGroup.Camera);
public static final Object CAMERA_LOCK = new Object();
public static synchronized void forceLoad() throws IOException {
if (libraryLoaded) return;
try {
File libDirectory = Path.of("lib/").toFile();
if (!libDirectory.exists()) {
Files.createDirectory(libDirectory.toPath()).toFile();
}
// We always extract the shared object (we could hash each so, but that's a lot of work)
URL resourceURL = LibCameraJNI.class.getResource("/nativelibraries/libphotonlibcamera.so");
File libFile = Path.of("lib/libphotonlibcamera.so").toFile();
try (InputStream in = resourceURL.openStream()) {
if (libFile.exists()) Files.delete(libFile.toPath());
Files.copy(in, libFile.toPath());
} catch (Exception e) {
logger.error("Could not extract the native library!");
}
System.load(libFile.getAbsolutePath());
libraryLoaded = true;
logger.info("Successfully loaded libpicam shared object");
} catch (UnsatisfiedLinkError e) {
logger.error("Couldn't load libpicam shared object");
e.printStackTrace();
}
}
public enum SensorModel {
Disconnected,
OV5647, // Picam v1
IMX219, // Picam v2
IMX477, // Picam HQ
OV9281,
OV7251,
Unknown;
public String getFriendlyName() {
switch (this) {
case Disconnected:
return "Disconnected Camera";
case OV5647:
return "Camera Module v1";
case IMX219:
return "Camera Module v2";
case IMX477:
return "HQ Camera";
case OV9281:
return "OV9281";
case OV7251:
return "OV7251";
case Unknown:
default:
return "Unknown Camera";
}
}
}
public static SensorModel getSensorModel() {
int model = getSensorModelRaw();
return SensorModel.values()[model];
}
public static boolean isSupported() {
return libraryLoaded
// && getSensorModel() != PicamJNI.SensorModel.Disconnected
// && Platform.isRaspberryPi()
&& isLibraryWorking();
}
private static native boolean isLibraryWorking();
public static native int getSensorModelRaw();
// ======================================================== //
/**
* Creates a new runner with a given width/height/fps
*
* @param width Camera video mode width in pixels
* @param height Camera video mode height in pixels
* @param fps Camera video mode FPS
* @return success of creating a camera object
*/
public static native boolean createCamera(int width, int height, int rotation);
/**
* Starts the camera thresholder and display threads running. Make sure that this function is
* called syncronously with stopCamera and returnFrame!
*/
public static native boolean startCamera();
/** Stops the camera runner. Make sure to call prior to destroying the camera! */
public static native boolean stopCamera();
// Destroy all native resources associated with a camera. Ensure stop is called prior!
public static native boolean destroyCamera();
// ======================================================== //
// Set thresholds on [0..1]
public static native boolean setThresholds(
double hl, double sl, double vl, double hu, double su, double vu, boolean hueInverted);
public static native boolean setAutoExposure(boolean doAutoExposure);
// Exposure time, in microseconds
public static native boolean setExposure(int exposureUs);
// Set brighness on [-1, 1]
public static native boolean setBrightness(double brightness);
// Unknown ranges for red and blue AWB gain
public static native boolean setAwbGain(double red, double blue);
/**
* Get the time when the first pixel exposure was started, in the same timebase as libcamera gives
* the frame capture time. Units are nanoseconds.
*/
public static native long getFrameCaptureTime();
/**
* Get the current time, in the same timebase as libcamera gives the frame capture time. Units are
* nanoseconds.
*/
public static native long getLibcameraTimestamp();
public static native long setFramesToCopy(boolean copyIn, boolean copyOut);
// Analog gain multiplier to apply to all color channels, on [1, Big Number]
public static native boolean setAnalogGain(double analog);
/** Block until a new frame is avaliable from native code. */
public static native boolean awaitNewFrame();
/**
* Get a pointer to the most recent color mat generated. Call this immediatly after awaitNewFrame,
* and call onlly once per new frame!
*/
public static native long takeColorFrame();
/**
* Get a pointer to the most recent processed mat generated. Call this immediatly after
* awaitNewFrame, and call onlly once per new frame!
*/
public static native long takeProcessedFrame();
/**
* Set the GPU processing type we should do. Enum of [none, HSV, greyscale, adaptive threshold].
*/
public static native boolean setGpuProcessType(int type);
public static native int getGpuProcessType();
// /** Release a frame pointer back to the libcamera driver code to be filled again */
// public static native long returnFrame(long frame);
}

163
photon-core/src/main/java/org/photonvision/raspi/PicamJNI.java
View File

@@ -1,163 +0,0 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.raspi;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import org.photonvision.common.hardware.PiVersion;
import org.photonvision.common.hardware.Platform;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
public class PicamJNI {
private static boolean libraryLoaded = false;
private static boolean enabled =
false; // TODO once we've sorted out what apriltags needs to be doing, we can bring this back?
private static Logger logger = new Logger(PicamJNI.class, LogGroup.Camera);
public enum SensorModel {
Disconnected,
OV5647, // Picam v1
IMX219, // Picam v2
IMX477, // Picam HQ
Unknown;
public String getFriendlyName() {
switch (this) {
case Disconnected:
return "Disconnected Camera";
case OV5647:
return "Camera Module v1";
case IMX219:
return "Camera Module v2";
case IMX477:
return "HQ Camera";
case Unknown:
default:
return "Unknown Camera";
}
}
}
public static synchronized void forceLoad() throws IOException {
if (libraryLoaded || !Platform.isRaspberryPi()) return;
try {
File libDirectory = Path.of("lib/").toFile();
if (!libDirectory.exists()) {
Files.createDirectory(libDirectory.toPath()).toFile();
}
// We always extract the shared object (we could hash each so, but that's a lot of work)
URL resourceURL = PicamJNI.class.getResource("/nativelibraries/libpicam.so");
File libFile = Path.of("lib/libpicam.so").toFile();
try (InputStream in = resourceURL.openStream()) {
if (libFile.exists()) Files.delete(libFile.toPath());
Files.copy(in, libFile.toPath());
} catch (Exception e) {
logger.error("Could not extract the native library!");
}
System.load(libFile.getAbsolutePath());
libraryLoaded = true;
logger.info("Successfully loaded libpicam shared object");
} catch (UnsatisfiedLinkError e) {
logger.error("Couldn't load libpicam shared object");
e.printStackTrace();
}
}
public static boolean isSupported() {
var piVer = PiVersion.getPiVersion();
boolean hwSupported =
(piVer == PiVersion.PI_3
|| piVer == PiVersion.COMPUTE_MODULE_3
|| piVer == PiVersion.ZERO_2_W);
return libraryLoaded
&& enabled
&& isVCSMSupported()
&& getSensorModel() != SensorModel.Disconnected
&& hwSupported;
}
public static SensorModel getSensorModel() {
switch (getSensorModelRaw().toLowerCase()) {
case "":
return SensorModel.Disconnected;
case "ov5647":
return SensorModel.OV5647;
case "imx219":
return SensorModel.IMX219;
case "imx477":
return SensorModel.IMX477;
default:
return SensorModel.Unknown;
}
}
private static native String getSensorModelRaw();
// This is the main thing we need that isn't supported on Pi 4s, which makes it a good check
private static native boolean isVCSMSupported();
// Everything here is static because multiple picams are unsupported at the hardware level
/**
* Called once for each video mode change. Starts a native thread running MMAL that stays alive
* until destroyCamera is called.
*
* @return true on error.
*/
public static native boolean createCamera(int width, int height, int fps);
/**
* Destroys MMAL and EGL contexts. Called once for each video mode change *before* createCamera.
*
* @return true on error.
*/
public static native boolean destroyCamera();
public static native void setThresholds(
double hL, double sL, double vL, double hU, double sU, double vU);
public static native void setInvertHue(boolean shouldInvert);
public static native boolean setExposure(int exposure);
public static native boolean setBrightness(int brightness);
// This adjusts the analog gain (normalized to 0-100); ignores the digital gain
public static native boolean setGain(int gain);
// Adjusts the auto white balance gains, which are normalized 0-100 in the native code
public static native boolean setAwbGain(int red, int blue);
public static native boolean setRotation(int rotation);
public static native void setShouldCopyColor(boolean shouldCopyColor);
public static native long getFrameLatency();
public static native long grabFrame(boolean shouldReturnColor);
}

220
photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSettables.java Normal file
View File

@@ -0,0 +1,220 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.camera;
import edu.wpi.first.cscore.VideoMode;
import edu.wpi.first.math.Pair;
import java.util.HashMap;
import org.photonvision.common.configuration.CameraConfiguration;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.raspi.LibCameraJNI;
import org.photonvision.vision.camera.LibcameraGpuSource.FPSRatedVideoMode;
import org.photonvision.vision.opencv.ImageRotationMode;
import org.photonvision.vision.processes.VisionSourceSettables;
public class LibcameraGpuSettables extends VisionSourceSettables {
private FPSRatedVideoMode currentVideoMode;
private double lastExposure = 50;
private int lastBrightness = 50;
private boolean lastExposureMode;
private int lastGain = 50;
private Pair<Integer, Integer> lastAwbGains = new Pair<>(18, 18);
private boolean m_initialized = false;
private ImageRotationMode m_rotationMode;
public void setRotation(ImageRotationMode rotationMode) {
if (rotationMode != m_rotationMode) {
m_rotationMode = rotationMode;
setVideoModeInternal(getCurrentVideoMode());
}
}
public LibcameraGpuSettables(CameraConfiguration configuration) {
super(configuration);
videoModes = new HashMap<>();
LibCameraJNI.SensorModel sensorModel = LibCameraJNI.getSensorModel();
if (sensorModel == LibCameraJNI.SensorModel.IMX219) {
// Settings for the IMX219 sensor, which is used on the Pi Camera Module v2
videoModes.put(
0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 120, 120, .39));
videoModes.put(
1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39));
videoModes.put(
2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
videoModes.put(
3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, .39));
// TODO: fix 1280x720 in the native code and re-add it
videoModes.put(
4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, .53));
videoModes.put(
5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 2, 2464 / 2, 15, 20, 1));
videoModes.put(
6, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 4, 2464 / 4, 15, 20, 1));
} else if (sensorModel == LibCameraJNI.SensorModel.OV9281) {
videoModes.put(
0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 800, 60, 60, 1));
videoModes.put(
1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280 / 2, 800 / 2, 60, 60, 1));
videoModes.put(
2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39));
videoModes.put(
3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
} else {
if (sensorModel == LibCameraJNI.SensorModel.IMX477) {
LibcameraGpuSource.logger.warn(
"It appears you are using a Pi HQ Camera. This camera is not officially supported. You will have to set your camera FOV differently based on resolution.");
} else if (sensorModel == LibCameraJNI.SensorModel.Unknown) {
LibcameraGpuSource.logger.warn(
"You have an unknown sensor connected to your Pi over CSI! This is likely a bug. If it is not, then you will have to set your camera FOV differently based on resolution.");
}
// Settings for the OV5647 sensor, which is used by the Pi Camera Module v1
videoModes.put(0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 90, 90, 1));
videoModes.put(1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 85, 90, 1));
videoModes.put(
2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 960, 720, 45, 49, 0.74));
// Half the size of the active areas on the OV5647
videoModes.put(
3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 2592 / 2, 1944 / 2, 20, 20, 1));
videoModes.put(
4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91));
videoModes.put(
5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72));
}
// TODO need to add more video modes for new sensors here
currentVideoMode = (FPSRatedVideoMode) videoModes.get(0);
}
@Override
public double getFOV() {
return getCurrentVideoMode().fovMultiplier * getConfiguration().FOV;
}
@Override
public void setAutoExposure(boolean cameraAutoExposure) {
lastExposureMode = cameraAutoExposure;
// TODO (Matt) -- call LibCameraJNI's auto exposure function, when that exists
LibCameraJNI.setAutoExposure(cameraAutoExposure);
}
@Override
public void setExposure(double exposure) {
// Todo (Chris) - for now, handle auto exposure by using -1
if (exposure < 0.0) {
exposure = -1;
}
// TODO convert to uS
lastExposure = exposure;
var success = LibCameraJNI.setExposure((int) Math.round(exposure) * 800);
if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera exposure");
}
@Override
public void setBrightness(int brightness) {
lastBrightness = brightness;
double realBrightness = MathUtils.map(brightness, 0.0, 100.0, -1.0, 1.0);
var success = LibCameraJNI.setBrightness(realBrightness);
if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera brightness");
}
@Override
public void setGain(int gain) {
lastGain = gain;
// TODO units here seem odd -- 5ish seems legit? So divide by 10
var success = LibCameraJNI.setAnalogGain(gain / 10.0);
if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera gain");
}
@Override
public void setRedGain(int red) {
lastAwbGains = Pair.of(red, lastAwbGains.getSecond());
setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
}
@Override
public void setBlueGain(int blue) {
lastAwbGains = Pair.of(lastAwbGains.getFirst(), blue);
setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
}
public void setAwbGain(int red, int blue) {
var success = LibCameraJNI.setAwbGain(red / 10.0, blue / 10.0);
if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera AWB gains");
}
@Override
public FPSRatedVideoMode getCurrentVideoMode() {
return currentVideoMode;
}
@Override
protected void setVideoModeInternal(VideoMode videoMode) {
var mode = (FPSRatedVideoMode) videoMode;
// We need to make sure that other threads don't try to do anything funny while we're recreating
// the camera
synchronized (LibCameraJNI.CAMERA_LOCK) {
boolean success = false;
if (m_initialized) {
success |= LibCameraJNI.stopCamera();
success |= LibCameraJNI.destroyCamera();
}
// if (!success) {
// throw new RuntimeException(
// "Couldn't destroy a zero copy Pi Camera while switching video modes");
// }
System.out.println("Starting camera");
success |=
LibCameraJNI.createCamera(
mode.width, mode.height, (m_rotationMode == ImageRotationMode.DEG_180 ? 180 : 0));
success |= LibCameraJNI.startCamera();
if (!success) {
throw new RuntimeException(
"Couldn't create a zero copy Pi Camera while switching video modes");
}
m_initialized = true;
}
// We don't store last settings on the native side, and when you change video mode these get
// reset on MMAL's end
setExposure(lastExposure);
setAutoExposure(lastExposureMode);
setBrightness(lastBrightness);
setGain(lastGain);
setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
LibCameraJNI.setFramesToCopy(true, true);
currentVideoMode = mode;
}
@Override
public HashMap<Integer, VideoMode> getAllVideoModes() {
return videoModes;
}
}

87
photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSource.java Normal file
View File

@@ -0,0 +1,87 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.camera;
import edu.wpi.first.cscore.VideoMode;
import org.photonvision.common.configuration.CameraConfiguration;
import org.photonvision.common.configuration.ConfigManager;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.frame.provider.LibcameraGpuFrameProvider;
import org.photonvision.vision.processes.VisionSource;
import org.photonvision.vision.processes.VisionSourceSettables;
public class LibcameraGpuSource extends VisionSource {
static final Logger logger = new Logger(LibcameraGpuSource.class, LogGroup.Camera);
private final LibcameraGpuSettables settables;
private final LibcameraGpuFrameProvider frameProvider;
public LibcameraGpuSource(CameraConfiguration configuration) {
super(configuration);
if (configuration.cameraType != CameraType.ZeroCopyPicam) {
throw new IllegalArgumentException(
"GPUAcceleratedPicamSource only accepts CameraConfigurations with type Picam");
}
settables = new LibcameraGpuSettables(configuration);
frameProvider = new LibcameraGpuFrameProvider(settables);
}
@Override
public FrameProvider getFrameProvider() {
return frameProvider;
}
@Override
public VisionSourceSettables getSettables() {
return settables;
}
/**
* On the OV5649 the actual FPS we want to request from the GPU can be higher than the FPS that we
* can do after processing. On the IMX219 these FPSes match pretty closely, except for the
* 1280x720 mode. We use this to present a rated FPS to the user that's lower than the actual FPS
* we request from the GPU. This is important for setting user expectations, and is also used by
* the frontend to detect and explain FPS drops. This class should ONLY be used by Picam video
* modes! This is to make sure it shows up nice in the frontend
*/
public static class FPSRatedVideoMode extends VideoMode {
public final int fpsActual;
public final double fovMultiplier;
public FPSRatedVideoMode(
PixelFormat pixelFormat,
int width,
int height,
int ratedFPS,
int actualFPS,
double fovMultiplier) {
super(pixelFormat, width, height, ratedFPS);
this.fpsActual = actualFPS;
this.fovMultiplier = fovMultiplier;
}
}
@Override
public boolean isVendorCamera() {
return ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV();
}
}

234
photon-core/src/main/java/org/photonvision/vision/camera/ZeroCopyPicamSource.java
View File

@@ -1,234 +0,0 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.camera;
import edu.wpi.first.cscore.VideoMode;
import edu.wpi.first.math.Pair;
import java.util.HashMap;
import org.photonvision.common.configuration.CameraConfiguration;
import org.photonvision.common.configuration.ConfigManager;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.frame.provider.AcceleratedPicamFrameProvider;
import org.photonvision.vision.processes.VisionSource;
import org.photonvision.vision.processes.VisionSourceSettables;
public class ZeroCopyPicamSource extends VisionSource {
private static final Logger logger = new Logger(ZeroCopyPicamSource.class, LogGroup.Camera);
private final VisionSourceSettables settables;
private final AcceleratedPicamFrameProvider frameProvider;
public ZeroCopyPicamSource(CameraConfiguration configuration) {
super(configuration);
if (configuration.cameraType != CameraType.ZeroCopyPicam) {
throw new IllegalArgumentException(
"GPUAcceleratedPicamSource only accepts CameraConfigurations with type Picam");
}
settables = new PicamSettables(configuration);
frameProvider = new AcceleratedPicamFrameProvider(settables);
}
@Override
public FrameProvider getFrameProvider() {
return frameProvider;
}
@Override
public VisionSourceSettables getSettables() {
return settables;
}
/**
* On the OV5649 the actual FPS we want to request from the GPU can be higher than the FPS that we
* can do after processing. On the IMX219 these FPSes match pretty closely, except for the
* 1280x720 mode. We use this to present a rated FPS to the user that's lower than the actual FPS
* we request from the GPU. This is important for setting user expectations, and is also used by
* the frontend to detect and explain FPS drops. This class should ONLY be used by Picam video
* modes! This is to make sure it shows up nice in the frontend
*/
public static class FPSRatedVideoMode extends VideoMode {
public final int fpsActual;
public final double fovMultiplier;
public FPSRatedVideoMode(
PixelFormat pixelFormat,
int width,
int height,
int ratedFPS,
int actualFPS,
double fovMultiplier) {
super(pixelFormat, width, height, ratedFPS);
this.fpsActual = actualFPS;
this.fovMultiplier = fovMultiplier;
}
}
public static class PicamSettables extends VisionSourceSettables {
private FPSRatedVideoMode currentVideoMode;
private double lastExposure = 50;
private int lastBrightness = 50;
private boolean lastExposureMode;
private int lastGain = 50;
private Pair<Integer, Integer> lastAwbGains = new Pair(18, 18);
public PicamSettables(CameraConfiguration configuration) {
super(configuration);
videoModes = new HashMap<>();
PicamJNI.SensorModel sensorModel = PicamJNI.getSensorModel();
if (sensorModel == PicamJNI.SensorModel.IMX219) {
// Settings for the IMX219 sensor, which is used on the Pi Camera Module v2
videoModes.put(
0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 120, 120, .39));
videoModes.put(
1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39));
videoModes.put(
2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
videoModes.put(
3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, .39));
// TODO: fix 1280x720 in the native code and re-add it
videoModes.put(
4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, .53));
} else {
if (sensorModel == PicamJNI.SensorModel.IMX477) {
logger.warn(
"It appears you are using a Pi HQ Camera. This camera is not officially supported. You will have to set your camera FOV differently based on resolution.");
} else if (sensorModel == PicamJNI.SensorModel.Unknown) {
logger.warn(
"You have an unknown sensor connected to your Pi over CSI! This is likely a bug. If it is not, then you will have to set your camera FOV differently based on resolution.");
}
// Settings for the OV5647 sensor, which is used by the Pi Camera Module v1
videoModes.put(
0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 90, 90, 1));
videoModes.put(
1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, 1));
videoModes.put(
2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 85, 90, 1));
videoModes.put(
3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, 1));
videoModes.put(
4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 960, 720, 45, 49, 0.74));
videoModes.put(
5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91));
videoModes.put(
6, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72));
}
currentVideoMode = (FPSRatedVideoMode) videoModes.get(0);
}
@Override
public double getFOV() {
return getCurrentVideoMode().fovMultiplier * getConfiguration().FOV;
}
@Override
public void setAutoExposure(boolean cameraAutoExposure) {
lastExposureMode = cameraAutoExposure;
// TODO (Matt) -- call PicamJNI's auto exposure function, when that exists
}
@Override
public void setExposure(double exposure) {
// Todo (Chris) - for now, handle auto exposure by using 100% exposure
if (exposure < 0.0) {
exposure = 100.0;
}
lastExposure = exposure;
var failure = PicamJNI.setExposure((int) Math.round(exposure));
if (failure) logger.warn("Couldn't set Pi Camera exposure");
}
@Override
public void setBrightness(int brightness) {
lastBrightness = brightness;
var failure = PicamJNI.setBrightness(brightness);
if (failure) logger.warn("Couldn't set Pi Camera brightness");
}
@Override
public void setGain(int gain) {
lastGain = gain;
var failure = PicamJNI.setGain(gain);
if (failure) logger.warn("Couldn't set Pi Camera gain");
}
@Override
public void setRedGain(int red) {
lastAwbGains = Pair.of(red, lastAwbGains.getSecond());
setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
}
@Override
public void setBlueGain(int blue) {
lastAwbGains = Pair.of(lastAwbGains.getFirst(), blue);
setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
}
public void setAwbGain(int red, int blue) {
var failure = PicamJNI.setAwbGain(red, blue);
if (failure) logger.warn("Couldn't set Pi Camera AWB gains");
}
@Override
public FPSRatedVideoMode getCurrentVideoMode() {
return currentVideoMode;
}
@Override
protected void setVideoModeInternal(VideoMode videoMode) {
var mode = (FPSRatedVideoMode) videoMode;
var failure = PicamJNI.destroyCamera();
if (failure)
throw new RuntimeException(
"Couldn't destroy a zero copy Pi Camera while switching video modes");
failure = PicamJNI.createCamera(mode.width, mode.height, mode.fpsActual);
if (failure)
throw new RuntimeException(
"Couldn't create a zero copy Pi Camera while switching video modes");
// We don't store last settings on the native side, and when you change video mode these get
// reset on MMAL's end
setExposure(lastExposure);
setAutoExposure(lastExposureMode);
setBrightness(lastBrightness);
setGain(lastGain);
setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
currentVideoMode = mode;
}
@Override
public HashMap<Integer, VideoMode> getAllVideoModes() {
return videoModes;
}
}
@Override
public boolean isVendorCamera() {
return ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV();
}
}

54
photon-core/src/main/java/org/photonvision/vision/frame/Frame.java
View File

@@ -17,52 +17,58 @@
package org.photonvision.vision.frame;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Size;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.opencv.Releasable;
public class Frame implements Releasable {
public final long timestampNanos;
public final CVMat image;
// Frame should at _least_ contain the thresholded frame, and sometimes the color image
public final CVMat colorImage;
public final CVMat processedImage;
public final FrameThresholdType type;
public final FrameStaticProperties frameStaticProperties;
public Frame(CVMat image, long timestampNanos, FrameStaticProperties frameStaticProperties) {
this.image = image;
public Frame(
CVMat color,
CVMat processed,
FrameThresholdType type,
long timestampNanos,
FrameStaticProperties frameStaticProperties) {
this.colorImage = color;
this.processedImage = processed;
this.type = type;
this.timestampNanos = timestampNanos;
this.frameStaticProperties = frameStaticProperties;
}
public Frame(CVMat image, FrameStaticProperties frameStaticProperties) {
this(image, MathUtils.wpiNanoTime(), frameStaticProperties);
public Frame(
CVMat color,
CVMat processed,
FrameThresholdType processType,
FrameStaticProperties frameStaticProperties) {
this(color, processed, processType, MathUtils.wpiNanoTime(), frameStaticProperties);
}
public Frame() {
this(new CVMat(), MathUtils.wpiNanoTime(), new FrameStaticProperties(0, 0, 0, null));
}
public static Frame emptyFrame(int width, int height) {
return new Frame(
new CVMat(Mat.zeros(new Size(width, height), CvType.CV_8UC3)),
this(
new CVMat(),
new CVMat(),
FrameThresholdType.NONE,
MathUtils.wpiNanoTime(),
new FrameStaticProperties(width, height, 0, null));
new FrameStaticProperties(0, 0, 0, null));
}
public void copyTo(Frame destFrame) {
image.getMat().copyTo(destFrame.image.getMat());
}
public static Frame copyFromAndRelease(Frame frame) {
var mat = new CVMat();
frame.image.copyTo(mat);
frame.release();
return new Frame(mat, frame.timestampNanos, frame.frameStaticProperties);
colorImage.getMat().copyTo(destFrame.colorImage.getMat());
processedImage.getMat().copyTo(destFrame.processedImage.getMat());
}
@Override
public void release() {
image.release();
colorImage.release();
processedImage.release();
}
}

14
photon-core/src/main/java/org/photonvision/vision/frame/FrameProvider.java
View File

@@ -18,7 +18,21 @@
package org.photonvision.vision.frame;
import java.util.function.Supplier;
import org.photonvision.vision.opencv.ImageRotationMode;
import org.photonvision.vision.pipe.impl.HSVPipe;
public interface FrameProvider extends Supplier<Frame> {
String getName();
/** Ask the camera to produce a certain kind of processed image (eg HSV or greyscale) */
public void requestFrameThresholdType(FrameThresholdType type);
/** Ask the camera to rotate frames it outputs */
public void requestFrameRotation(ImageRotationMode rotationMode);
/** Ask the camera to provide either the input, output, or both frames. */
public void requestFrameCopies(boolean copyInput, boolean copyOutput);
/** Ask the camera to rotate frames it outputs */
public void requestHsvSettings(HSVPipe.HSVParams params);
}

22
photon-core/src/main/java/org/photonvision/vision/frame/provider/NetworkFrameProvider.java → photon-core/src/main/java/org/photonvision/vision/frame/FrameThresholdType.java
View File

@@ -15,22 +15,10 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.frame.provider;
package org.photonvision.vision.frame;
import org.apache.commons.lang3.NotImplementedException;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameProvider;
public class NetworkFrameProvider implements FrameProvider {
private int count = 0;
@Override
public Frame get() {
throw new NotImplementedException("");
}
@Override
public String getName() {
return "NetworkFrameProvider" + count++;
}
public enum FrameThresholdType {
NONE,
HSV,
GREYSCALE,
}

10
photon-core/src/main/java/org/photonvision/vision/frame/consumer/FileSaveFrameConsumer.java
View File

@@ -31,9 +31,9 @@ import org.photonvision.common.dataflow.networktables.NetworkTablesManager;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.common.util.TimedTaskManager;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.opencv.CVMat;
public class FileSaveFrameConsumer implements Consumer<Frame> {
public class FileSaveFrameConsumer implements Consumer<CVMat> {
// Formatters to generate unique, timestamped file names
private static String FILE_PATH = ConfigManager.getInstance().getImageSavePath().toString();
private static String FILE_EXTENSION = ".jpg";
@@ -62,8 +62,8 @@ public class FileSaveFrameConsumer implements Consumer<Frame> {
this.logger = new Logger(FileSaveFrameConsumer.class, this.camNickname, LogGroup.General);
}
public void accept(Frame frame) {
if (frame != null && !frame.image.getMat().empty()) {
public void accept(CVMat image) {
if (image != null && image.getMat() != null && !image.getMat().empty()) {
if (lock.tryLock()) {
boolean curCommand = entry.get(false);
if (curCommand && !prevCommand) {
@@ -78,7 +78,7 @@ public class FileSaveFrameConsumer implements Consumer<Frame> {
+ tf.format(now)
+ FILE_EXTENSION;
Imgcodecs.imwrite(savefile, frame.image.getMat());
Imgcodecs.imwrite(savefile, image.getMat());
// Help the user a bit - set the NT entry back to false after 500ms
TimedTaskManager.getInstance().addOneShotTask(this::resetCommand, CMD_RESET_TIME_MS);

8
photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java
View File

@@ -28,7 +28,7 @@ import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.imgproc.Imgproc;
import org.photonvision.common.util.ColorHelper;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.opencv.CVMat;
public class MJPGFrameConsumer {
public static final Mat EMPTY_MAT = new Mat(60, 15 * 7, CvType.CV_8UC3);
@@ -167,9 +167,9 @@ public class MJPGFrameConsumer {
this(name, 320, 240, port);
}
public void accept(Frame frame) {
if (frame != null && !frame.image.getMat().empty()) {
cvSource.putFrame(frame.image.getMat());
public void accept(CVMat image) {
if (image != null && !image.getMat().empty()) {
cvSource.putFrame(image.getMat());
// Make sure our disabled framerate limiting doesn't get confused
isDisabled = false;

60
photon-core/src/main/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProvider.java
View File

@@ -1,60 +0,0 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.frame.provider;
import org.opencv.core.Mat;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.processes.VisionSourceSettables;
public class AcceleratedPicamFrameProvider implements FrameProvider {
private final VisionSourceSettables settables;
private CVMat mat;
public AcceleratedPicamFrameProvider(VisionSourceSettables visionSettables) {
this.settables = visionSettables;
var vidMode = settables.getCurrentVideoMode();
var failure = PicamJNI.createCamera(vidMode.width, vidMode.height, vidMode.fps);
if (failure) {
failure = PicamJNI.destroyCamera();
if (failure) throw new RuntimeException("Couldn't destroy Pi camera after init failure!");
throw new RuntimeException(
"Couldn't initialize zero copy Pi camera; check stdout for native code logs");
}
}
@Override
public String getName() {
return "AcceleratedPicamFrameProvider";
}
@Override
public Frame get() {
long matHandle = PicamJNI.grabFrame(false);
mat = new CVMat(new Mat(matHandle));
return new Frame(
mat,
MathUtils.wpiNanoTime() - PicamJNI.getFrameLatency(),
settables.getFrameStaticProperties());
}
}

124
photon-core/src/main/java/org/photonvision/vision/frame/provider/CpuImageProcessor.java Normal file
View File

@@ -0,0 +1,124 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.frame.provider;
import org.photonvision.common.util.numbers.IntegerCouple;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.frame.FrameStaticProperties;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.opencv.ImageRotationMode;
import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
import org.photonvision.vision.pipe.impl.GrayscalePipe;
import org.photonvision.vision.pipe.impl.HSVPipe;
import org.photonvision.vision.pipe.impl.RotateImagePipe;
public abstract class CpuImageProcessor implements FrameProvider {
protected class CapturedFrame {
CVMat colorImage;
FrameStaticProperties staticProps;
long captureTimestamp;
public CapturedFrame(
CVMat colorImage, FrameStaticProperties staticProps, long captureTimestampNanos) {
this.colorImage = colorImage;
this.staticProps = staticProps;
this.captureTimestamp = captureTimestampNanos;
}
}
private final HSVPipe m_hsvPipe = new HSVPipe();
private final RotateImagePipe m_rImagePipe = new RotateImagePipe();
private final GrayscalePipe m_grayPipe = new GrayscalePipe();
FrameThresholdType m_processType;
private final Object m_mutex = new Object();
abstract CapturedFrame getInputMat();
public CpuImageProcessor() {
m_hsvPipe.setParams(
new HSVPipe.HSVParams(
new IntegerCouple(0, 180),
new IntegerCouple(0, 255),
new IntegerCouple(0, 255),
false));
}
@Override
public final Frame get() {
// TODO Auto-generated method stub
var input = getInputMat();
CVMat outputMat = null;
long sumNanos = 0;
{
CVPipeResult<Void> out = m_rImagePipe.run(input.colorImage.getMat());
sumNanos += out.nanosElapsed;
}
if (!input.colorImage.getMat().empty()) {
if (m_processType == FrameThresholdType.HSV) {
var hsvResult = m_hsvPipe.run(input.colorImage.getMat());
outputMat = new CVMat(hsvResult.output);
sumNanos += hsvResult.nanosElapsed;
} else if (m_processType == FrameThresholdType.GREYSCALE) {
var result = m_grayPipe.run(input.colorImage.getMat());
outputMat = new CVMat(result.output);
sumNanos += result.nanosElapsed;
} else {
outputMat = new CVMat();
}
} else {
System.out.println("Input was empty!");
outputMat = new CVMat();
}
return new Frame(
input.colorImage, outputMat, m_processType, input.captureTimestamp, input.staticProps);
}
@Override
public void requestFrameThresholdType(FrameThresholdType type) {
synchronized (m_mutex) {
this.m_processType = type;
}
}
@Override
public void requestFrameRotation(ImageRotationMode rotationMode) {
synchronized (m_mutex) {
m_rImagePipe.setParams(new RotateImagePipe.RotateImageParams(rotationMode));
}
}
/** Ask the camera to rotate frames it outputs */
public void requestHsvSettings(HSVPipe.HSVParams params) {
synchronized (m_mutex) {
m_hsvPipe.setParams(params);
}
}
@Override
public void requestFrameCopies(boolean copyInput, boolean copyOutput) {
// We don't actually do zero-copy, so this method is a no-op
return;
}
}

16
photon-core/src/main/java/org/photonvision/vision/frame/provider/FileFrameProvider.java
View File

@@ -22,8 +22,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import org.opencv.core.Mat;
import org.opencv.imgcodecs.Imgcodecs;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.frame.FrameStaticProperties;
import org.photonvision.vision.opencv.CVMat;
@@ -32,14 +32,14 @@ import org.photonvision.vision.opencv.CVMat;
* A {@link FrameProvider} that will read and provide an image from a {@link java.nio.file.Path
* path}.
*/
public class FileFrameProvider implements FrameProvider {
public class FileFrameProvider extends CpuImageProcessor {
public static final int MAX_FPS = 5;
private static int count = 0;
private final int thisIndex = count++;
private final Path path;
private final int millisDelay;
private final Frame originalFrame;
private final CVMat originalFrame;
private final FrameStaticProperties properties;
@@ -70,7 +70,7 @@ public class FileFrameProvider implements FrameProvider {
Mat rawImage = Imgcodecs.imread(path.toString());
if (rawImage.cols() > 0 && rawImage.rows() > 0) {
properties = new FrameStaticProperties(rawImage.width(), rawImage.height(), fov, calibration);
originalFrame = new Frame(new CVMat(rawImage), properties);
originalFrame = new CVMat(rawImage);
} else {
throw new RuntimeException("Image loading failed!");
}
@@ -97,9 +97,9 @@ public class FileFrameProvider implements FrameProvider {
}
@Override
public Frame get() {
Frame outputFrame = new Frame(new CVMat(), properties);
originalFrame.copyTo(outputFrame);
public CapturedFrame getInputMat() {
var out = new CVMat();
out.copyTo(originalFrame);
// block to keep FPS at a defined rate
if (System.currentTimeMillis() - lastGetMillis < millisDelay) {
@@ -111,7 +111,7 @@ public class FileFrameProvider implements FrameProvider {
}
lastGetMillis = System.currentTimeMillis();
return outputFrame;
return new CapturedFrame(out, properties, MathUtils.wpiNanoTime());
}
@Override

114
photon-core/src/main/java/org/photonvision/vision/frame/provider/LibcameraGpuFrameProvider.java Normal file
View File

@@ -0,0 +1,114 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.frame.provider;
import org.opencv.core.Mat;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.raspi.LibCameraJNI;
import org.photonvision.vision.camera.LibcameraGpuSettables;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.opencv.ImageRotationMode;
import org.photonvision.vision.pipe.impl.HSVPipe.HSVParams;
public class LibcameraGpuFrameProvider implements FrameProvider {
private final LibcameraGpuSettables settables;
public LibcameraGpuFrameProvider(LibcameraGpuSettables visionSettables) {
this.settables = visionSettables;
var vidMode = settables.getCurrentVideoMode();
settables.setVideoMode(vidMode);
}
@Override
public String getName() {
return "AcceleratedPicamFrameProvider";
}
int i = 0;
@Override
public Frame get() {
// We need to make sure that other threads don't try to change video modes while we're waiting
// for a frame
// System.out.println("GET!");
synchronized (LibCameraJNI.CAMERA_LOCK) {
var success = LibCameraJNI.awaitNewFrame();
if (!success) {
System.out.println("No new frame");
return new Frame();
}
var colorMat = new CVMat(new Mat(LibCameraJNI.takeColorFrame()));
var processedMat = new CVMat(new Mat(LibCameraJNI.takeProcessedFrame()));
// System.out.println("Color mat: " + colorMat.getMat().size());
// Imgcodecs.imwrite("color" + i + ".jpg", colorMat.getMat());
// Imgcodecs.imwrite("processed" + (i) + ".jpg", processedMat.getMat());
int itype = LibCameraJNI.getGpuProcessType();
FrameThresholdType type = FrameThresholdType.NONE;
if (itype < FrameThresholdType.values().length && itype >= 0) {
type = FrameThresholdType.values()[itype];
}
var now = LibCameraJNI.getLibcameraTimestamp();
var capture = LibCameraJNI.getFrameCaptureTime();
var latency = (now - capture);
return new Frame(
colorMat,
processedMat,
type,
MathUtils.wpiNanoTime() - latency,
settables.getFrameStaticProperties());
}
}
@Override
public void requestFrameThresholdType(FrameThresholdType type) {
LibCameraJNI.setGpuProcessType(type.ordinal());
}
@Override
public void requestFrameRotation(ImageRotationMode rotationMode) {
this.settables.setRotation(rotationMode);
}
@Override
public void requestHsvSettings(HSVParams params) {
LibCameraJNI.setThresholds(
params.getHsvLower().val[0] / 180.0,
params.getHsvLower().val[1] / 255.0,
params.getHsvLower().val[2] / 255.0,
params.getHsvUpper().val[0] / 180.0,
params.getHsvUpper().val[1] / 255.0,
params.getHsvUpper().val[2] / 255.0,
params.getHueInverted());
}
@Override
public void requestFrameCopies(boolean copyInput, boolean copyOutput) {
LibCameraJNI.setFramesToCopy(copyInput, copyOutput);
}
}

13
photon-core/src/main/java/org/photonvision/vision/frame/provider/USBFrameProvider.java
View File

@@ -19,12 +19,10 @@ package org.photonvision.vision.frame.provider;
import edu.wpi.first.cscore.CvSink;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.processes.VisionSourceSettables;
public class USBFrameProvider implements FrameProvider {
public class USBFrameProvider extends CpuImageProcessor {
private final CvSink cvSink;
@SuppressWarnings("SpellCheckingInspection")
@@ -38,18 +36,19 @@ public class USBFrameProvider implements FrameProvider {
}
@Override
public Frame get() {
public CapturedFrame getInputMat() {
var mat = new CVMat(); // We do this so that we don't fill a Mat in use by another thread
// This is from wpi::Now, or WPIUtilJNI.now()
long time =
cvSink.grabFrame(
mat.getMat()); // Units are microseconds, epoch is the same as the Unix epoch
cvSink.grabFrame(mat.getMat())
* 1000; // Units are microseconds, epoch is the same as the Unix epoch
// Sometimes CSCore gives us a zero frametime.
if (time <= 1e-6) {
time = MathUtils.wpiNanoTime();
}
return new Frame(mat, MathUtils.microsToNanos(time), settables.getFrameStaticProperties());
return new CapturedFrame(mat, settables.getFrameStaticProperties(), time);
}
@Override

12
photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java
View File

@@ -20,11 +20,11 @@ package org.photonvision.vision.pipe.impl;
import edu.wpi.first.apriltag.AprilTagDetection;
import edu.wpi.first.apriltag.AprilTagDetector;
import java.util.List;
import org.opencv.core.Mat;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.pipe.CVPipe;
public class AprilTagDetectionPipe
extends CVPipe<Mat, List<AprilTagDetection>, AprilTagDetectionPipeParams> {
extends CVPipe<CVMat, List<AprilTagDetection>, AprilTagDetectionPipeParams> {
private final AprilTagDetector m_detector = new AprilTagDetector();
boolean useNativePoseEst;
@@ -37,8 +37,12 @@ public class AprilTagDetectionPipe
}
@Override
protected List<AprilTagDetection> process(Mat in) {
var ret = m_detector.detect(in);
protected List<AprilTagDetection> process(CVMat in) {
if (in.getMat().empty()) {
return List.of();
}
var ret = m_detector.detect(in.getMat());
if (ret == null) {
return List.of();

4
photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterContoursPipe.java
View File

@@ -107,8 +107,8 @@ public class FilterContoursPipe
// Fullness Filtering.
double contourArea = contour.getArea();
double minFullness = params.getFullness().getFirst() * minAreaRect.size.area() / 100;
double maxFullness = params.getFullness().getSecond() * minAreaRect.size.area() / 100;
double minFullness = params.getFullness().getFirst() * minAreaRect.size.area() / 100.0;
double maxFullness = params.getFullness().getSecond() * minAreaRect.size.area() / 100.0;
if (contourArea <= minFullness || contourArea >= maxFullness) return;
// Aspect Ratio Filtering.

56
photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java
View File

@@ -24,12 +24,9 @@ import edu.wpi.first.math.geometry.Transform3d;
import edu.wpi.first.math.util.Units;
import java.util.ArrayList;
import java.util.List;
import org.opencv.core.Mat;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.vision.camera.CameraQuirk;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
import org.photonvision.vision.pipe.impl.*;
import org.photonvision.vision.pipe.impl.AprilTagPoseEstimatorPipe.AprilTagPoseEstimatorPipeParams;
@@ -39,17 +36,19 @@ import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;
@SuppressWarnings("DuplicatedCode")
public class AprilTagPipeline extends CVPipeline<CVPipelineResult, AprilTagPipelineSettings> {
private final RotateImagePipe rotateImagePipe = new RotateImagePipe();
private final GrayscalePipe grayscalePipe = new GrayscalePipe();
private final AprilTagDetectionPipe aprilTagDetectionPipe = new AprilTagDetectionPipe();
private final AprilTagPoseEstimatorPipe poseEstimatorPipe = new AprilTagPoseEstimatorPipe();
private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.GREYSCALE;
public AprilTagPipeline() {
super(PROCESSING_TYPE);
settings = new AprilTagPipelineSettings();
}
public AprilTagPipeline(AprilTagPipelineSettings settings) {
super(PROCESSING_TYPE);
this.settings = settings;
}
@@ -58,15 +57,11 @@ public class AprilTagPipeline extends CVPipeline<CVPipelineResult, AprilTagPipel
// Sanitize thread count - not supported to have fewer than 1 threads
settings.threads = Math.max(1, settings.threads);
RotateImagePipe.RotateImageParams rotateImageParams =
new RotateImagePipe.RotateImageParams(settings.inputImageRotationMode);
rotateImagePipe.setParams(rotateImageParams);
if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && PicamJNI.isSupported()) {
// TODO: Picam grayscale
PicamJNI.setRotation(settings.inputImageRotationMode.value);
PicamJNI.setShouldCopyColor(true); // need the color image to grayscale
}
// if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && LibCameraJNI.isSupported()) {
// // TODO: Picam grayscale
// LibCameraJNI.setRotation(settings.inputImageRotationMode.value);
// // LibCameraJNI.setShouldCopyColor(true); // need the color image to grayscale
// }
// TODO (HACK): tag width is Fun because it really belongs in the "target model"
// We need the tag width for the JNI to figure out target pose, but we need a
@@ -131,33 +126,18 @@ public class AprilTagPipeline extends CVPipeline<CVPipelineResult, AprilTagPipel
protected CVPipelineResult process(Frame frame, AprilTagPipelineSettings settings) {
long sumPipeNanosElapsed = 0L;
CVPipeResult<Mat> grayscalePipeResult;
Mat rawInputMat;
boolean inputSingleChannel = frame.image.getMat().channels() == 1;
if (inputSingleChannel) {
rawInputMat = new Mat(PicamJNI.grabFrame(true));
frame.image.getMat().release(); // release the 8bit frame ASAP.
} else {
rawInputMat = frame.image.getMat();
var rotateImageResult = rotateImagePipe.run(rawInputMat);
sumPipeNanosElapsed += rotateImageResult.nanosElapsed;
}
var inputFrame = new Frame(new CVMat(rawInputMat), frameStaticProperties);
grayscalePipeResult = grayscalePipe.run(rawInputMat);
sumPipeNanosElapsed += grayscalePipeResult.nanosElapsed;
var outputFrame = new Frame(new CVMat(grayscalePipeResult.output), frameStaticProperties);
List<TrackedTarget> targetList;
CVPipeResult<List<AprilTagDetection>> tagDetectionPipeResult;
// Use the solvePNP Enabled flag to enable native pose estimation
aprilTagDetectionPipe.setNativePoseEstimationEnabled(settings.solvePNPEnabled);
tagDetectionPipeResult = aprilTagDetectionPipe.run(grayscalePipeResult.output);
if (frame.type != FrameThresholdType.GREYSCALE) {
// TODO so all cameras should give us ADAPTIVE_THRESH -- how should we handle if not?
return new CVPipelineResult(0, 0, List.of());
}
CVPipeResult<List<AprilTagDetection>> tagDetectionPipeResult;
tagDetectionPipeResult = aprilTagDetectionPipe.run(frame.processedImage);
sumPipeNanosElapsed += tagDetectionPipeResult.nanosElapsed;
targetList = new ArrayList<>();
@@ -194,6 +174,6 @@ public class AprilTagPipeline extends CVPipeline<CVPipelineResult, AprilTagPipel
var fpsResult = calculateFPSPipe.run(null);
var fps = fpsResult.output;
return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, outputFrame, inputFrame);
return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, frame);
}
}

20
photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java
View File

@@ -17,10 +17,10 @@
package org.photonvision.vision.pipeline;
import java.util.List;
import org.photonvision.vision.camera.QuirkyCamera;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameStaticProperties;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.pipeline.result.CVPipelineResult;
public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelineSettings> {
@@ -28,6 +28,16 @@ public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelin
protected FrameStaticProperties frameStaticProperties;
protected QuirkyCamera cameraQuirks;
private final FrameThresholdType thresholdType;
public CVPipeline(FrameThresholdType thresholdType) {
this.thresholdType = thresholdType;
}
public FrameThresholdType getThresholdType() {
return thresholdType;
}
protected void setPipeParams(
FrameStaticProperties frameStaticProperties, S settings, QuirkyCamera cameraQuirks) {
this.settings = settings;
@@ -55,10 +65,10 @@ public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelin
}
setPipeParams(frame.frameStaticProperties, settings, cameraQuirks);
if (frame.image.getMat().empty()) {
//noinspection unchecked
return (R) new CVPipelineResult(0, 0, List.of(), frame);
}
// if (frame.image.getMat().empty()) {
// //noinspection unchecked
// return (R) new CVPipelineResult(0, 0, List.of(), frame);
// }
R result = process(frame, settings);
result.setImageCaptureTimestampNanos(frame.timestampNanos);

6
photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipelineSettings.java
View File

@@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import java.util.Objects;
import org.photonvision.vision.frame.FrameDivisor;
import org.photonvision.vision.opencv.ImageFlipMode;
import org.photonvision.vision.opencv.ImageRotationMode;
@JsonTypeInfo(
@@ -37,7 +36,6 @@ import org.photonvision.vision.opencv.ImageRotationMode;
public class CVPipelineSettings implements Cloneable {
public int pipelineIndex = 0;
public PipelineType pipelineType = PipelineType.DriverMode;
public ImageFlipMode inputImageFlipMode = ImageFlipMode.NONE;
public ImageRotationMode inputImageRotationMode = ImageRotationMode.DEG_0;
public String pipelineNickname = "New Pipeline";
public boolean cameraAutoExposure = false;
@@ -70,7 +68,6 @@ public class CVPipelineSettings implements Cloneable {
&& cameraVideoModeIndex == that.cameraVideoModeIndex
&& ledMode == that.ledMode
&& pipelineType == that.pipelineType
&& inputImageFlipMode == that.inputImageFlipMode
&& inputImageRotationMode == that.inputImageRotationMode
&& pipelineNickname.equals(that.pipelineNickname)
&& streamingFrameDivisor == that.streamingFrameDivisor
@@ -83,7 +80,6 @@ public class CVPipelineSettings implements Cloneable {
return Objects.hash(
pipelineIndex,
pipelineType,
inputImageFlipMode,
inputImageRotationMode,
pipelineNickname,
cameraExposure,
@@ -115,8 +111,6 @@ public class CVPipelineSettings implements Cloneable {
+ pipelineIndex
+ ", pipelineType="
+ pipelineType
+ ", inputImageFlipMode="
+ inputImageFlipMode
+ ", inputImageRotationMode="
+ inputImageRotationMode
+ ", pipelineNickname='"

31
photon-core/src/main/java/org/photonvision/vision/pipeline/Calibrate3dPipeline.java
View File

@@ -34,10 +34,9 @@ import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.common.util.SerializationUtils;
import org.photonvision.common.util.file.FileUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
import org.photonvision.vision.camera.CameraQuirk;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
import org.photonvision.vision.pipe.impl.CalculateFPSPipe;
@@ -71,11 +70,14 @@ public class Calibrate3dPipeline
// Path to save images
private final Path imageDir = ConfigManager.getInstance().getCalibDir();
private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE;
public Calibrate3dPipeline() {
this(12);
}
public Calibrate3dPipeline(int minSnapshots) {
super(PROCESSING_TYPE);
this.settings = new Calibration3dPipelineSettings();
this.foundCornersList = new ArrayList<>();
this.minSnapshots = minSnapshots;
@@ -93,26 +95,18 @@ public class Calibrate3dPipeline
new Size(frameStaticProperties.imageWidth, frameStaticProperties.imageHeight));
calibrate3dPipe.setParams(calibratePipeParams);
if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && PicamJNI.isSupported()) {
PicamJNI.setRotation(settings.inputImageRotationMode.value);
PicamJNI.setShouldCopyColor(true);
}
// if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && LibCameraJNI.isSupported()) {
// LibCameraJNI.setRotation(settings.inputImageRotationMode.value);
// // LibCameraJNI.setShouldCopyColor(true);
// }
}
@Override
protected CVPipelineResult process(Frame frame, Calibration3dPipelineSettings settings) {
Mat inputColorMat = frame.image.getMat();
if (inputColorMat.channels() == 1
&& cameraQuirks.hasQuirk(CameraQuirk.PiCam)
&& PicamJNI.isSupported()) {
long colorMatPtr = PicamJNI.grabFrame(true);
if (colorMatPtr == 0) throw new RuntimeException("Got null Mat from GPU Picam driver");
inputColorMat = new Mat(colorMatPtr);
}
Mat inputColorMat = frame.colorImage.getMat();
if (this.calibrating) {
return new CVPipelineResult(
0, 0, null, new Frame(new CVMat(inputColorMat), frame.frameStaticProperties));
return new CVPipelineResult(0, 0, null, frame);
}
long sumPipeNanosElapsed = 0L;
@@ -141,14 +135,15 @@ public class Calibrate3dPipeline
}
}
frame.image.release();
frame.release();
// Return the drawn chessboard if corners are found, if not, then return the input image.
return new CVPipelineResult(
sumPipeNanosElapsed,
fps, // Unused but here in case
Collections.emptyList(),
new Frame(outputColorCVMat, frame.frameStaticProperties));
new Frame(
new CVMat(), outputColorCVMat, FrameThresholdType.NONE, frame.frameStaticProperties));
}
public void deleteSavedImages() {

7
photon-core/src/main/java/org/photonvision/vision/pipeline/Calibration3dPipelineSettings.java
View File

@@ -27,4 +27,11 @@ public class Calibration3dPipelineSettings extends AdvancedPipelineSettings {
public double gridSize = Units.inchesToMeters(1.0);
public Size resolution = new Size(640, 480);
public Calibration3dPipelineSettings() {
super();
this.inputShouldShow = true;
this.outputShouldShow = true;
}
}

78
photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java
View File

@@ -21,12 +21,9 @@ import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.vision.camera.CameraQuirk;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.opencv.*;
import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
import org.photonvision.vision.pipe.impl.*;
@@ -37,9 +34,7 @@ import org.photonvision.vision.target.TrackedTarget;
@SuppressWarnings({"DuplicatedCode"})
public class ColoredShapePipeline
extends CVPipeline<CVPipelineResult, ColoredShapePipelineSettings> {
private final RotateImagePipe rotateImagePipe = new RotateImagePipe();
private final ErodeDilatePipe erodeDilatePipe = new ErodeDilatePipe();
private final HSVPipe hsvPipe = new HSVPipe();
private final SpeckleRejectPipe speckleRejectPipe = new SpeckleRejectPipe();
private final FindContoursPipe findContoursPipe = new FindContoursPipe();
private final FindPolygonPipe findPolygonPipe = new FindPolygonPipe();
@@ -56,11 +51,15 @@ public class ColoredShapePipeline
private final Point[] rectPoints = new Point[4];
private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.HSV;
public ColoredShapePipeline() {
super(PROCESSING_TYPE);
settings = new ColoredShapePipelineSettings();
}
public ColoredShapePipeline(ColoredShapePipelineSettings settings) {
super(PROCESSING_TYPE);
this.settings = settings;
}
@@ -73,29 +72,6 @@ public class ColoredShapePipeline
settings.offsetDualPointB,
settings.offsetDualPointBArea);
RotateImagePipe.RotateImageParams rotateImageParams =
new RotateImagePipe.RotateImageParams(settings.inputImageRotationMode);
rotateImagePipe.setParams(rotateImageParams);
if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && PicamJNI.isSupported()) {
PicamJNI.setThresholds(
settings.hsvHue.getFirst() / 180d,
settings.hsvSaturation.getFirst() / 255d,
settings.hsvValue.getFirst() / 255d,
settings.hsvHue.getSecond() / 180d,
settings.hsvSaturation.getSecond() / 255d,
settings.hsvValue.getSecond() / 255d);
PicamJNI.setInvertHue(settings.hueInverted);
PicamJNI.setRotation(settings.inputImageRotationMode.value);
PicamJNI.setShouldCopyColor(settings.inputShouldShow);
} else {
var hsvParams =
new HSVPipe.HSVParams(
settings.hsvHue, settings.hsvSaturation, settings.hsvValue, settings.hueInverted);
hsvPipe.setParams(hsvParams);
}
ErodeDilatePipe.ErodeDilateParams erodeDilateParams =
new ErodeDilatePipe.ErodeDilateParams(settings.erode, settings.dilate, 5);
// TODO: add kernel size to pipeline settings
@@ -199,45 +175,14 @@ public class ColoredShapePipeline
protected CVPipelineResult process(Frame frame, ColoredShapePipelineSettings settings) {
long sumPipeNanosElapsed = 0L;
CVPipeResult<Mat> hsvPipeResult;
Mat rawInputMat;
if (frame.image.getMat().channels() != 1) {
var rotateImageResult = rotateImagePipe.run(frame.image.getMat());
sumPipeNanosElapsed = rotateImageResult.nanosElapsed;
rawInputMat = frame.image.getMat();
hsvPipeResult = hsvPipe.run(rawInputMat);
sumPipeNanosElapsed += hsvPipeResult.nanosElapsed;
} else {
// Try to copy the color frame.
long inputMatPtr = PicamJNI.grabFrame(true);
if (inputMatPtr != 0) {
// If we grabbed it (in color copy mode), make a new Mat of it
rawInputMat = new Mat(inputMatPtr);
} else {
// // Otherwise, use a blank/empty mat as placeholder
// rawInputMat = new Mat();
// Otherwise, the input mat is frame we got from the camera
rawInputMat = frame.image.getMat();
}
// We can skip a few steps if the image is single channel because we've already done them on
// the GPU
hsvPipeResult = new CVPipeResult<>();
hsvPipeResult.output = frame.image.getMat();
hsvPipeResult.nanosElapsed = MathUtils.wpiNanoTime() - frame.timestampNanos;
sumPipeNanosElapsed += hsvPipeResult.nanosElapsed;
}
// var erodeDilateResult = erodeDilatePipe.run(rawInputMat);
// sumPipeNanosElapsed += erodeDilateResult.nanosElapsed;
//
// CVPipeResult<Mat> hsvPipeResult = hsvPipe.run(rawInputMat);
// sumPipeNanosElapsed += hsvPipeResult.nanosElapsed;
CVPipeResult<List<Contour>> findContoursResult = findContoursPipe.run(hsvPipeResult.output);
CVPipeResult<List<Contour>> findContoursResult =
findContoursPipe.run(frame.processedImage.getMat());
sumPipeNanosElapsed += findContoursResult.nanosElapsed;
CVPipeResult<List<Contour>> speckleRejectResult =
@@ -247,7 +192,7 @@ public class ColoredShapePipeline
List<CVShape> shapes = null;
if (settings.contourShape == ContourShape.Circle) {
CVPipeResult<List<CVShape>> findCirclesResult =
findCirclesPipe.run(Pair.of(hsvPipeResult.output, speckleRejectResult.output));
findCirclesPipe.run(Pair.of(frame.processedImage.getMat(), speckleRejectResult.output));
sumPipeNanosElapsed += findCirclesResult.nanosElapsed;
shapes = findCirclesResult.output;
} else {
@@ -293,11 +238,6 @@ public class ColoredShapePipeline
var fpsResult = calculateFPSPipe.run(null);
var fps = fpsResult.output;
return new CVPipelineResult(
sumPipeNanosElapsed,
fps,
targetList,
new Frame(new CVMat(hsvPipeResult.output), frame.frameStaticProperties),
new Frame(new CVMat(rawInputMat), frame.frameStaticProperties));
return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, frame);
}
}

34
photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java
View File

@@ -19,12 +19,11 @@ package org.photonvision.vision.pipeline;
import java.util.List;
import org.apache.commons.lang3.tuple.Pair;
import org.opencv.core.Mat;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.raspi.LibCameraJNI;
import org.photonvision.vision.camera.CameraQuirk;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.pipe.impl.CalculateFPSPipe;
import org.photonvision.vision.pipe.impl.Draw2dCrosshairPipe;
import org.photonvision.vision.pipe.impl.ResizeImagePipe;
@@ -38,10 +37,18 @@ public class DriverModePipeline
private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
private final ResizeImagePipe resizeImagePipe = new ResizeImagePipe();
private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE;
public DriverModePipeline() {
super(PROCESSING_TYPE);
settings = new DriverModePipelineSettings();
}
public DriverModePipeline(DriverModePipelineSettings settings) {
super(PROCESSING_TYPE);
this.settings = settings;
}
@Override
protected void setPipeParamsImpl() {
RotateImagePipe.RotateImageParams rotateImageParams =
@@ -56,25 +63,19 @@ public class DriverModePipeline
resizeImagePipe.setParams(
new ResizeImagePipe.ResizeImageParams(settings.streamingFrameDivisor));
if (PicamJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
PicamJNI.setRotation(settings.inputImageRotationMode.value);
PicamJNI.setShouldCopyColor(true);
}
// if (LibCameraJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
// LibCameraJNI.setRotation(settings.inputImageRotationMode.value);
// LibCameraJNI.setShouldCopyColor(true);
// }
}
@Override
public DriverModePipelineResult process(Frame frame, DriverModePipelineSettings settings) {
long totalNanos = 0;
boolean accelerated = PicamJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam);
boolean accelerated = LibCameraJNI.isSupported() && cameraQuirks.hasQuirk(CameraQuirk.PiCam);
// apply pipes
var inputMat = frame.image.getMat();
if (inputMat.channels() == 1 && accelerated) {
long colorMatPtr = PicamJNI.grabFrame(true);
if (colorMatPtr == 0) throw new RuntimeException("Got null Mat from GPU Picam driver");
frame.image.release();
inputMat = new Mat(colorMatPtr);
}
var inputMat = frame.colorImage.getMat();
totalNanos += resizeImagePipe.run(inputMat).nanosElapsed;
@@ -91,9 +92,10 @@ public class DriverModePipeline
var fpsResult = calculateFPSPipe.run(null);
var fps = fpsResult.output;
// Flip-flop input and output in the Frame
return new DriverModePipelineResult(
MathUtils.nanosToMillis(totalNanos),
fps,
new Frame(new CVMat(inputMat), frame.frameStaticProperties));
new Frame(frame.processedImage, frame.colorImage, frame.type, frame.frameStaticProperties));
}
}

17
photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java
View File

@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.commons.lang3.tuple.Pair;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameStaticProperties;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.opencv.ContourShape;
import org.photonvision.vision.opencv.DualOffsetValues;
import org.photonvision.vision.pipe.impl.*;
@@ -99,18 +98,19 @@ public class OutputStreamPipeline {
}
public CVPipelineResult process(
Frame inputFrame,
Frame outputFrame,
Frame inputAndOutputFrame,
AdvancedPipelineSettings settings,
List<TrackedTarget> targetsToDraw) {
setPipeParams(inputFrame.frameStaticProperties, settings);
var inMat = inputFrame.image.getMat();
var outMat = outputFrame.image.getMat();
setPipeParams(inputAndOutputFrame.frameStaticProperties, settings);
var inMat = inputAndOutputFrame.colorImage.getMat();
var outMat = inputAndOutputFrame.processedImage.getMat();
long sumPipeNanosElapsed = 0L;
// Resize both in place before doing any conversion
sumPipeNanosElapsed += pipeProfileNanos[0] = resizeImagePipe.run(inMat).nanosElapsed;
boolean inEmpty = inMat.empty();
if (!inEmpty)
sumPipeNanosElapsed += pipeProfileNanos[0] = resizeImagePipe.run(inMat).nanosElapsed;
sumPipeNanosElapsed += pipeProfileNanos[1] = resizeImagePipe.run(outMat).nanosElapsed;
// Convert single-channel HSV output mat to 3-channel BGR in preparation for streaming
@@ -183,7 +183,6 @@ public class OutputStreamPipeline {
sumPipeNanosElapsed,
fps, // Unused but here just in case
targetsToDraw,
new Frame(new CVMat(outMat), outputFrame.frameStaticProperties),
new Frame(new CVMat(inMat), inputFrame.frameStaticProperties));
inputAndOutputFrame);
}
}

96
photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java
View File

@@ -18,12 +18,8 @@
package org.photonvision.vision.pipeline;
import java.util.List;
import org.opencv.core.Mat;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.vision.camera.CameraQuirk;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.opencv.Contour;
import org.photonvision.vision.opencv.DualOffsetValues;
import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
@@ -36,8 +32,6 @@ import org.photonvision.vision.target.TrackedTarget;
/** Represents a pipeline for tracking retro-reflective targets. */
@SuppressWarnings({"DuplicatedCode"})
public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectivePipelineSettings> {
private final RotateImagePipe rotateImagePipe = new RotateImagePipe();
private final HSVPipe hsvPipe = new HSVPipe();
private final FindContoursPipe findContoursPipe = new FindContoursPipe();
private final SpeckleRejectPipe speckleRejectPipe = new SpeckleRejectPipe();
private final FilterContoursPipe filterContoursPipe = new FilterContoursPipe();
@@ -50,11 +44,15 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
private final long[] pipeProfileNanos = new long[PipelineProfiler.ReflectivePipeCount];
private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.HSV;
public ReflectivePipeline() {
super(PROCESSING_TYPE);
settings = new ReflectivePipelineSettings();
}
public ReflectivePipeline(ReflectivePipelineSettings settings) {
super(PROCESSING_TYPE);
this.settings = settings;
}
@@ -67,27 +65,28 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
settings.offsetDualPointB,
settings.offsetDualPointBArea);
var rotateImageParams = new RotateImagePipe.RotateImageParams(settings.inputImageRotationMode);
rotateImagePipe.setParams(rotateImageParams);
// var rotateImageParams = new
// RotateImagePipe.RotateImageParams(settings.inputImageRotationMode);
// rotateImagePipe.setParams(rotateImageParams);
if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && PicamJNI.isSupported()) {
PicamJNI.setThresholds(
settings.hsvHue.getFirst() / 180d,
settings.hsvSaturation.getFirst() / 255d,
settings.hsvValue.getFirst() / 255d,
settings.hsvHue.getSecond() / 180d,
settings.hsvSaturation.getSecond() / 255d,
settings.hsvValue.getSecond() / 255d);
PicamJNI.setInvertHue(settings.hueInverted);
PicamJNI.setRotation(settings.inputImageRotationMode.value);
PicamJNI.setShouldCopyColor(settings.inputShouldShow);
} else {
var hsvParams =
new HSVPipe.HSVParams(
settings.hsvHue, settings.hsvSaturation, settings.hsvValue, settings.hueInverted);
hsvPipe.setParams(hsvParams);
}
// if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && LibCameraJNI.isSupported()) {
// LibCameraJNI.setThresholds(
// settings.hsvHue.getFirst() / 180d,
// settings.hsvSaturation.getFirst() / 255d,
// settings.hsvValue.getFirst() / 255d,
// settings.hsvHue.getSecond() / 180d,
// settings.hsvSaturation.getSecond() / 255d,
// settings.hsvValue.getSecond() / 255d);
// // LibCameraJNI.setInvertHue(settings.hueInverted);
// LibCameraJNI.setRotation(settings.inputImageRotationMode.value);
// // LibCameraJNI.setShouldCopyColor(settings.inputShouldShow);
// } else {
// var hsvParams =
// new HSVPipe.HSVParams(
// settings.hsvHue, settings.hsvSaturation, settings.hsvValue,
// settings.hueInverted);
// hsvPipe.setParams(hsvParams);
// }
var findContoursParams = new FindContoursPipe.FindContoursParams();
findContoursPipe.setParams(findContoursParams);
@@ -148,40 +147,8 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
public CVPipelineResult process(Frame frame, ReflectivePipelineSettings settings) {
long sumPipeNanosElapsed = 0L;
CVPipeResult<Mat> hsvPipeResult;
Mat rawInputMat;
if (frame.image.getMat().channels() != 1) {
var rotateImageResult = rotateImagePipe.run(frame.image.getMat());
sumPipeNanosElapsed += pipeProfileNanos[0] = rotateImageResult.nanosElapsed;
rawInputMat = frame.image.getMat();
hsvPipeResult = hsvPipe.run(rawInputMat);
sumPipeNanosElapsed += hsvPipeResult.nanosElapsed;
pipeProfileNanos[1] = pipeProfileNanos[1] = hsvPipeResult.nanosElapsed;
} else {
// Try to copy the color frame.
long inputMatPtr = PicamJNI.grabFrame(true);
if (inputMatPtr != 0) {
// If we grabbed it (in color copy mode), make a new Mat of it
rawInputMat = new Mat(inputMatPtr);
} else {
// Otherwise, the input mat is frame we got from the camera
rawInputMat = frame.image.getMat();
// // Otherwise, use a blank/empty mat as placeholder
// rawInputMat = new Mat();
}
// We can skip a few steps if the image is single channel because we've already done them on
// the GPU
hsvPipeResult = new CVPipeResult<>();
hsvPipeResult.output = frame.image.getMat();
hsvPipeResult.nanosElapsed = MathUtils.wpiNanoTime() - frame.timestampNanos;
sumPipeNanosElapsed = pipeProfileNanos[1] = hsvPipeResult.nanosElapsed;
}
CVPipeResult<List<Contour>> findContoursResult = findContoursPipe.run(hsvPipeResult.output);
CVPipeResult<List<Contour>> findContoursResult =
findContoursPipe.run(frame.processedImage.getMat());
sumPipeNanosElapsed += pipeProfileNanos[2] = findContoursResult.nanosElapsed;
CVPipeResult<List<Contour>> speckleRejectResult =
@@ -226,11 +193,6 @@ public class ReflectivePipeline extends CVPipeline<CVPipelineResult, ReflectiveP
PipelineProfiler.printReflectiveProfile(pipeProfileNanos);
return new CVPipelineResult(
sumPipeNanosElapsed,
fps,
targetList,
new Frame(new CVMat(hsvPipeResult.output), frame.frameStaticProperties),
new Frame(new CVMat(rawInputMat), frame.frameStaticProperties));
return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, frame);
}
}

21
photon-core/src/main/java/org/photonvision/vision/pipeline/result/CVPipelineResult.java
View File

@@ -29,26 +29,19 @@ public class CVPipelineResult implements Releasable {
public final double processingNanos;
public final double fps;
public final List<TrackedTarget> targets;
public final Frame outputFrame;
public final Frame inputFrame;
public final Frame inputAndOutputFrame;
public CVPipelineResult(
double processingNanos,
double fps,
List<TrackedTarget> targets,
Frame outputFrame,
Frame inputFrame) {
double processingNanos, double fps, List<TrackedTarget> targets, Frame inputFrame) {
this.processingNanos = processingNanos;
this.fps = fps;
this.targets = targets != null ? targets : Collections.emptyList();
this.outputFrame = outputFrame;
this.inputFrame = inputFrame;
this.inputAndOutputFrame = inputFrame;
}
public CVPipelineResult(
double processingNanos, double fps, List<TrackedTarget> targets, Frame outputFrame) {
this(processingNanos, fps, targets, outputFrame, null);
public CVPipelineResult(double processingNanos, double fps, List<TrackedTarget> targets) {
this(processingNanos, fps, targets, null);
}
public boolean hasTargets() {
@@ -59,8 +52,7 @@ public class CVPipelineResult implements Releasable {
for (TrackedTarget tt : targets) {
tt.release();
}
outputFrame.release();
if (inputFrame != null) inputFrame.release();
if (inputAndOutputFrame != null) inputAndOutputFrame.release();
}
/**
@@ -68,6 +60,7 @@ public class CVPipelineResult implements Releasable {
* the latency is relative to the time at which this method is called. Waiting to call this method
* will change the latency this method returns.
*/
@Deprecated
public double getLatencyMillis() {
var now = MathUtils.wpiNanoTime();
return MathUtils.nanosToMillis(now - imageCaptureTimestampNanos);

82
photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java
View File

@@ -21,6 +21,7 @@ import edu.wpi.first.cscore.VideoException;
import edu.wpi.first.math.util.Units;
import io.javalin.websocket.WsContext;
import java.util.*;
import java.util.function.BiConsumer;
import org.photonvision.common.configuration.CameraConfiguration;
import org.photonvision.common.configuration.ConfigManager;
import org.photonvision.common.configuration.PhotonConfiguration;
@@ -33,12 +34,11 @@ import org.photonvision.common.hardware.HardwareManager;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.common.util.SerializationUtils;
import org.photonvision.common.util.java.TriConsumer;
import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
import org.photonvision.vision.camera.CameraQuirk;
import org.photonvision.vision.camera.LibcameraGpuSource;
import org.photonvision.vision.camera.QuirkyCamera;
import org.photonvision.vision.camera.USBCameraSource;
import org.photonvision.vision.camera.ZeroCopyPicamSource;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.consumer.FileSaveFrameConsumer;
import org.photonvision.vision.pipeline.AdvancedPipelineSettings;
@@ -65,7 +65,7 @@ public class VisionModule {
private final StreamRunnable streamRunnable;
private final LinkedList<CVPipelineResultConsumer> resultConsumers = new LinkedList<>();
// Raw result consumers run before any drawing has been done by the OutputStreamPipeline
private final LinkedList<TriConsumer<Frame, Frame, List<TrackedTarget>>> streamResultConsumers =
private final LinkedList<BiConsumer<Frame, List<TrackedTarget>>> streamResultConsumers =
new LinkedList<>();
private final NTDataPublisher ntConsumer;
private final UIDataPublisher uiDataConsumer;
@@ -93,7 +93,7 @@ public class VisionModule {
// Find quirks for the current camera
if (visionSource instanceof USBCameraSource) {
cameraQuirks = ((USBCameraSource) visionSource).cameraQuirks;
} else if (visionSource instanceof ZeroCopyPicamSource) {
} else if (visionSource instanceof LibcameraGpuSource) {
cameraQuirks = QuirkyCamera.ZeroCopyPiCamera;
} else {
cameraQuirks = QuirkyCamera.DefaultCamera;
@@ -190,17 +190,29 @@ public class VisionModule {
}
private void recreateStreamResultConsumers() {
streamResultConsumers.add((in, out, tgts) -> inputFrameSaver.accept(in));
streamResultConsumers.add((in, out, tgts) -> outputFrameSaver.accept(out));
streamResultConsumers.add((in, out, tgts) -> inputVideoStreamer.accept(in));
streamResultConsumers.add((in, out, tgts) -> outputVideoStreamer.accept(out));
streamResultConsumers.add(
(frame, tgts) -> {
if (frame != null) inputFrameSaver.accept(frame.colorImage);
});
streamResultConsumers.add(
(frame, tgts) -> {
if (frame != null) outputFrameSaver.accept(frame.processedImage);
});
streamResultConsumers.add(
(frame, tgts) -> {
if (frame != null) inputVideoStreamer.accept(frame.colorImage);
});
streamResultConsumers.add(
(frame, tgts) -> {
if (frame != null) outputVideoStreamer.accept(frame.processedImage);
});
}
private class StreamRunnable extends Thread {
private final OutputStreamPipeline outputStreamPipeline;
private final Object frameLock = new Object();
private Frame inputFrame, outputFrame;
private Frame latestFrame;
private AdvancedPipelineSettings settings = new AdvancedPipelineSettings();
private List<TrackedTarget> targets = new ArrayList<>();
@@ -211,42 +223,35 @@ public class VisionModule {
}
public void updateData(
Frame inputFrame,
Frame outputFrame,
AdvancedPipelineSettings settings,
List<TrackedTarget> targets) {
Frame inputOutputFrame, AdvancedPipelineSettings settings, List<TrackedTarget> targets) {
synchronized (frameLock) {
if (shouldRun && this.inputFrame != null && this.outputFrame != null) {
if (shouldRun && this.latestFrame != null) {
logger.trace("Fell behind; releasing last unused Mats");
this.inputFrame.release();
this.outputFrame.release();
this.latestFrame.release();
}
this.inputFrame = inputFrame;
this.outputFrame = outputFrame;
this.latestFrame = inputOutputFrame;
this.settings = settings;
this.targets = targets;
shouldRun =
inputFrame != null
&& !inputFrame.image.getMat().empty()
&& outputFrame != null
&& !outputFrame.image.getMat().empty();
shouldRun = inputOutputFrame != null;
// && inputOutputFrame.colorImage != null
// && !inputOutputFrame.colorImage.getMat().empty()
// && inputOutputFrame.processedImage != null
// && !inputOutputFrame.processedImage.getMat().empty();
}
}
@Override
public void run() {
while (true) {
final Frame inputFrame, outputFrame;
final Frame m_frame;
final AdvancedPipelineSettings settings;
final List<TrackedTarget> targets;
final boolean shouldRun;
synchronized (frameLock) {
inputFrame = this.inputFrame;
outputFrame = this.outputFrame;
this.inputFrame = null;
this.outputFrame = null;
m_frame = this.latestFrame;
this.latestFrame = null;
settings = this.settings;
targets = this.targets;
@@ -256,17 +261,15 @@ public class VisionModule {
}
if (shouldRun) {
try {
CVPipelineResult osr =
outputStreamPipeline.process(inputFrame, outputFrame, settings, targets);
consumeResults(inputFrame, osr.outputFrame, targets);
CVPipelineResult osr = outputStreamPipeline.process(m_frame, settings, targets);
consumeResults(m_frame, targets);
} catch (Exception e) {
// Never die
logger.error("Exception while running stream runnable!", e);
}
try {
inputFrame.release();
outputFrame.release();
m_frame.release();
} catch (Exception e) {
logger.error("Exception freeing frames", e);
}
@@ -496,7 +499,7 @@ public class VisionModule {
internalMap.put("fps", videoModes.get(k).fps);
internalMap.put(
"pixelFormat",
((videoModes.get(k) instanceof ZeroCopyPicamSource.FPSRatedVideoMode)
((videoModes.get(k) instanceof LibcameraGpuSource.FPSRatedVideoMode)
? "kPicam"
: videoModes.get(k).pixelFormat.toString())
.substring(1)); // Remove the k prefix
@@ -546,16 +549,15 @@ public class VisionModule {
consumePipelineResult(result);
// Pipelines like DriverMode and Calibrate3dPipeline have null output frames
if (result.inputFrame != null
if (result.inputAndOutputFrame != null
&& (pipelineManager.getCurrentPipelineSettings() instanceof AdvancedPipelineSettings)) {
streamRunnable.updateData(
result.inputFrame,
result.outputFrame,
result.inputAndOutputFrame,
(AdvancedPipelineSettings) pipelineManager.getCurrentPipelineSettings(),
result.targets);
// The streamRunnable manages releasing in this case
} else {
consumeResults(result.inputFrame, result.outputFrame, result.targets);
consumeResults(result.inputAndOutputFrame, result.targets);
result.release();
// In this case we don't bother with a separate streaming thread and we release
@@ -569,9 +571,9 @@ public class VisionModule {
}
/** Consume stream/target results, no rate limiting applied */
private void consumeResults(Frame inputFrame, Frame outputFrame, List<TrackedTarget> targets) {
private void consumeResults(Frame frame, List<TrackedTarget> targets) {
for (var c : streamResultConsumers) {
c.accept(inputFrame, outputFrame, targets);
c.accept(frame, targets);
}
}

27
photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java
View File

@@ -22,8 +22,9 @@ import java.util.function.Supplier;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.vision.camera.QuirkyCamera;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameProvider;
import org.photonvision.vision.pipe.impl.HSVPipe;
import org.photonvision.vision.pipeline.AdvancedPipelineSettings;
import org.photonvision.vision.pipeline.CVPipeline;
import org.photonvision.vision.pipeline.result.CVPipelineResult;
@@ -32,7 +33,7 @@ import org.photonvision.vision.pipeline.result.CVPipelineResult;
public class VisionRunner {
private final Logger logger;
private final Thread visionProcessThread;
private final Supplier<Frame> frameSupplier;
private final FrameProvider frameSupplier;
private final Supplier<CVPipeline> pipelineSupplier;
private final Consumer<CVPipelineResult> pipelineResultConsumer;
private final QuirkyCamera cameraQuirks;
@@ -69,8 +70,29 @@ public class VisionRunner {
private void update() {
while (!Thread.interrupted()) {
var pipeline = pipelineSupplier.get();
// Tell our camera implementation here what kind of pre-processing we need it to be doing
// (pipeline-dependent). I kinda hate how much leak this has...
// TODO would a callback object be a better fit?
var wantedProcessType = pipeline.getThresholdType();
frameSupplier.requestFrameThresholdType(wantedProcessType);
var settings = pipeline.getSettings();
if (settings instanceof AdvancedPipelineSettings) {
var advanced = (AdvancedPipelineSettings) settings;
var hsvParams =
new HSVPipe.HSVParams(
advanced.hsvHue, advanced.hsvSaturation, advanced.hsvValue, advanced.hueInverted);
// TODO who should deal with preventing this from happening _every single loop_?
frameSupplier.requestHsvSettings(hsvParams);
}
frameSupplier.requestFrameRotation(settings.inputImageRotationMode);
frameSupplier.requestFrameCopies(settings.inputShouldShow, settings.outputShouldShow);
// Grab the new camera frame
var frame = frameSupplier.get();
// There's no guarantee the processing type change will occur this tick, so pipelines should
// check themselves
try {
var pipelineResult = pipeline.run(frame, cameraQuirks);
pipelineResultConsumer.accept(pipelineResult);
@@ -78,6 +100,7 @@ public class VisionRunner {
logger.error("Exception on loop " + loopCount);
ex.printStackTrace();
}
loopCount++;
}
}

43
photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java
View File

@@ -27,14 +27,15 @@ import org.photonvision.common.configuration.CameraConfiguration;
import org.photonvision.common.configuration.ConfigManager;
import org.photonvision.common.dataflow.DataChangeService;
import org.photonvision.common.dataflow.events.OutgoingUIEvent;
import org.photonvision.common.hardware.Platform;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;
import org.photonvision.common.util.TimedTaskManager;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.raspi.LibCameraJNI;
import org.photonvision.vision.camera.CameraQuirk;
import org.photonvision.vision.camera.CameraType;
import org.photonvision.vision.camera.LibcameraGpuSource;
import org.photonvision.vision.camera.USBCameraSource;
import org.photonvision.vision.camera.ZeroCopyPicamSource;
public class VisionSourceManager {
private static final Logger logger = new Logger(VisionSourceManager.class, LogGroup.Camera);
@@ -252,8 +253,14 @@ public class VisionSourceManager {
logger.info("Creating a new camera config for camera " + uniqueName);
// HACK -- for picams, we want to use the camera model
String nickname = uniqueName;
if (isCsiCamera(info)) {
nickname = LibCameraJNI.getSensorModel().toString();
}
CameraConfiguration configuration =
new CameraConfiguration(baseName, uniqueName, uniqueName, info.path);
new CameraConfiguration(baseName, uniqueName, nickname, info.path, info.otherPaths);
cameraConfigurations.add(configuration);
}
@@ -261,6 +268,11 @@ public class VisionSourceManager {
return cameraConfigurations;
}
private boolean isCsiCamera(UsbCameraInfo configuration) {
return (Arrays.stream(configuration.otherPaths).anyMatch(it -> it.contains("csi-video"))
|| cameraNameToBaseName(configuration.name).equals("unicam"));
}
private CameraConfiguration mergeInfoIntoConfig(CameraConfiguration cfg, UsbCameraInfo info) {
if (!cfg.path.equals(info.path)) {
logger.debug("Updating path config from " + cfg.path + " to " + info.path);
@@ -307,20 +319,23 @@ public class VisionSourceManager {
List<CameraConfiguration> camConfigs) {
var cameraSources = new ArrayList<VisionSource>();
for (var configuration : camConfigs) {
if (configuration.baseName.startsWith("mmal service") && PicamJNI.isSupported()) {
configuration.cameraType = CameraType.ZeroCopyPicam;
var piCamSrc = new ZeroCopyPicamSource(configuration);
System.out.println("Creating VisionSource for " + configuration);
// Picams should have csi-video in the path
boolean is_picam =
(Arrays.stream(configuration.otherPaths).anyMatch(it -> it.contains("csi-video"))
|| configuration.baseName.equals("unicam"));
boolean is_pi = Platform.isRaspberryPi();
if (is_picam && is_pi) {
configuration.cameraType = CameraType.ZeroCopyPicam;
var piCamSrc = new LibcameraGpuSource(configuration);
cameraSources.add(piCamSrc);
continue;
}
var newCam = new USBCameraSource(configuration);
if (!newCam.cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken)
&& !newCam.getSettables().videoModes.isEmpty()) {
cameraSources.add(newCam);
} else {
var newCam = new USBCameraSource(configuration);
if (!newCam.cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken)
&& !newCam.getSettables().videoModes.isEmpty()) {
cameraSources.add(newCam);
}
}
}
return cameraSources;

12
photon-core/src/main/java/org/photonvision/vision/videoStream/SocketVideoStream.java
View File

@@ -25,10 +25,10 @@ import org.opencv.core.MatOfByte;
import org.opencv.core.MatOfInt;
import org.opencv.imgcodecs.Imgcodecs;
import org.photonvision.common.util.math.MathUtils;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.consumer.MJPGFrameConsumer;
import org.photonvision.vision.opencv.CVMat;
public class SocketVideoStream implements Consumer<Frame> {
public class SocketVideoStream implements Consumer<CVMat> {
int portID = 0; // Align with cscore's port for unique identification of stream
MatOfByte jpegBytes = null;
@@ -55,7 +55,7 @@ public class SocketVideoStream implements Consumer<Frame> {
}
@Override
public void accept(Frame frame) {
public void accept(CVMat image) {
if (userCount > 0) {
if (jpegBytesLock
.tryLock()) { // we assume frames are coming in frequently. Just skip this frame if we're
@@ -63,12 +63,12 @@ public class SocketVideoStream implements Consumer<Frame> {
try {
// Does a single-shot frame recieve and convert to JPEG for efficency
// Will not capture/convert again until convertNextFrame() is called
if (frame != null && !frame.image.getMat().empty() && jpegBytes == null) {
if (image != null && !image.getMat().empty() && jpegBytes == null) {
frameWasConsumed = false;
jpegBytes = new MatOfByte();
Imgcodecs.imencode(
".jpg",
frame.image.getMat(),
image.getMat(),
jpegBytes,
new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 75));
}
@@ -81,7 +81,7 @@ public class SocketVideoStream implements Consumer<Frame> {
// Send the frame in an FPS-limited fashion
var now = MathUtils.wpiNanoTime();
if (now > nextFrameSendTime) {
oldSchoolServer.accept(frame);
oldSchoolServer.accept(image);
nextFrameSendTime = now + minFramePeriodNanos;
}
}

52
photon-core/src/test/java/org/photonvision/vision/frame/provider/AcceleratedPicamFrameProviderTest.java
View File

@@ -1,52 +0,0 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.frame.provider;
import java.io.IOException;
import org.junit.jupiter.api.Test;
import org.opencv.imgcodecs.Imgcodecs;
import org.photonvision.common.configuration.CameraConfiguration;
import org.photonvision.common.util.TestUtils;
import org.photonvision.raspi.PicamJNI;
import org.photonvision.vision.camera.ZeroCopyPicamSource;
public class AcceleratedPicamFrameProviderTest {
@Test
public void testGrabFrame() throws IOException {
PicamJNI.forceLoad();
if (!PicamJNI.isSupported()) return;
TestUtils.loadLibraries();
var frameProvider =
new AcceleratedPicamFrameProvider(
new ZeroCopyPicamSource.PicamSettables(new CameraConfiguration("f", "f", "f", "f")));
long lastTime = System.currentTimeMillis();
for (int i = 0; i < 10; i++) {
var frame = frameProvider.get();
System.out.println(frame.image.getMat().get(0, 0)[0]);
long time = System.currentTimeMillis();
System.out.println("dt (ms): " + (time - lastTime));
lastTime = time;
}
var mat = frameProvider.get().image.getMat();
Imgcodecs.imwrite("out.png", mat);
}
}

12
photon-core/src/test/java/org/photonvision/vision/frame/provider/FileFrameProviderTest.java
View File

@@ -48,14 +48,14 @@ public class FileFrameProviderTest {
Frame goodFrame = goodFrameProvider.get();
int goodFrameCols = goodFrame.image.getMat().cols();
int goodFrameRows = goodFrame.image.getMat().rows();
int goodFrameCols = goodFrame.colorImage.getMat().cols();
int goodFrameRows = goodFrame.colorImage.getMat().rows();
// 2019 Images are at 320x240
assertEquals(320, goodFrameCols);
assertEquals(240, goodFrameRows);
TestUtils.showImage(goodFrame.image.getMat(), "2019");
TestUtils.showImage(goodFrame.colorImage.getMat(), "2019");
var badFilePath = Paths.get("bad.jpg"); // this file does not exist
@@ -81,14 +81,14 @@ public class FileFrameProviderTest {
Frame goodFrame = goodFrameProvider.get();
int goodFrameCols = goodFrame.image.getMat().cols();
int goodFrameRows = goodFrame.image.getMat().rows();
int goodFrameCols = goodFrame.colorImage.getMat().cols();
int goodFrameRows = goodFrame.colorImage.getMat().rows();
// 2020 Images are at 640x480
assertEquals(640, goodFrameCols);
assertEquals(480, goodFrameRows);
TestUtils.showImage(goodFrame.image.getMat(), "2020");
TestUtils.showImage(goodFrame.colorImage.getMat(), "2020");
var badFilePath = Paths.get("bad.jpg"); // this file does not exist

38
photon-core/src/test/java/org/photonvision/vision/frame/provider/LibcameraTest.java Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.photonvision.vision.frame.provider;
import org.junit.jupiter.api.Test;
// import org.photonvision.raspi.LibCameraJNI;
public class LibcameraTest {
@Test
public void testBasic() {
// System.load("/home/pi/photon-libcamera-gl-driver/build/libphotonlibcamera.so");
// LibCameraJNI.createCamera(1920, 1080, 60);
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// }
// LibCameraJNI.startCamera();
// try {
// Thread.sleep(5000);
// } catch (InterruptedException e) {
// }
}
}

9
photon-core/src/test/java/org/photonvision/vision/pipeline/AprilTagTest.java
View File

@@ -52,6 +52,7 @@ public class AprilTagTest {
TestUtils.getApriltagImagePath(TestUtils.ApriltagTestImages.kTag1_640_480, false),
TestUtils.WPI2020Image.FOV,
TestUtils.get2020LifeCamCoeffs(false));
frameProvider.requestFrameThresholdType(pipeline.getThresholdType());
CVPipelineResult pipelineResult;
try {
@@ -65,12 +66,10 @@ public class AprilTagTest {
// Draw on input
var outputPipe = new OutputStreamPipeline();
outputPipe.process(
pipelineResult.inputFrame,
pipelineResult.outputFrame,
pipeline.getSettings(),
pipelineResult.targets);
pipelineResult.inputAndOutputFrame, pipeline.getSettings(), pipelineResult.targets);
TestUtils.showImage(pipelineResult.inputFrame.image.getMat(), "Pipeline output", 999999);
TestUtils.showImage(
pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999);
// these numbers are not *accurate*, but they are known and expected
var pose = pipelineResult.targets.get(0).getBestCameraToTarget3d();

12
photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java
View File

@@ -38,6 +38,7 @@ import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
import org.photonvision.vision.camera.QuirkyCamera;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameStaticProperties;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.pipe.impl.Calibrate3dPipe;
import org.photonvision.vision.pipe.impl.FindBoardCornersPipe;
@@ -103,9 +104,11 @@ public class Calibrate3dPipeTest {
var frame =
new Frame(
new CVMat(Imgcodecs.imread(file.getAbsolutePath())),
new CVMat(),
FrameThresholdType.NONE,
new FrameStaticProperties(640, 480, 60, null));
var output = calibration3dPipeline.run(frame, QuirkyCamera.DefaultCamera);
// TestUtils.showImage(output.outputFrame.image.getMat());
// TestUtils.showImage(output.inputAndOutputFrame.processedImage.getMat());
output.release();
frame.release();
}
@@ -119,6 +122,8 @@ public class Calibrate3dPipeTest {
var frame =
new Frame(
new CVMat(Imgcodecs.imread(directoryListing[0].getAbsolutePath())),
new CVMat(),
FrameThresholdType.NONE,
new FrameStaticProperties(640, 480, 60, null));
calibration3dPipeline.run(frame, QuirkyCamera.DefaultCamera).release();
frame.release();
@@ -266,10 +271,13 @@ public class Calibrate3dPipeTest {
var frame =
new Frame(
new CVMat(Imgcodecs.imread(file.getAbsolutePath())),
new CVMat(),
FrameThresholdType.NONE,
new FrameStaticProperties((int) imgRes.width, (int) imgRes.height, 67, null));
var output = calibration3dPipeline.run(frame, QuirkyCamera.DefaultCamera);
// TestUtils.showImage(output.outputFrame.image.getMat(), file.getName(), 1);
// TestUtils.showImage(output.inputAndOutputFrame.processedImage.getMat(), file.getName(),
// 1);
output.release();
frame.release();
}

4
photon-core/src/test/java/org/photonvision/vision/pipeline/CirclePNPTest.java
View File

@@ -111,11 +111,13 @@ public class CirclePNPTest {
TestUtils.getPowercellImagePath(TestUtils.PowercellTestImages.kPowercell_test_6, false),
TestUtils.WPI2020Image.FOV,
TestUtils.get2020LifeCamCoeffs(true));
frameProvider.requestFrameThresholdType(pipeline.getThresholdType());
CVPipelineResult pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera);
printTestResults(pipelineResult);
TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output", 999999);
TestUtils.showImage(
pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999);
}
private static void continuouslyRunPipeline(Frame frame, ReflectivePipelineSettings settings) {

12
photon-core/src/test/java/org/photonvision/vision/pipeline/ColoredShapePipelineTest.java
View File

@@ -33,7 +33,8 @@ public class ColoredShapePipelineTest {
pipeline.settings = settings;
CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera);
TestUtils.showImage(
colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Triangle.");
colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(),
"Pipeline output: Triangle.");
printTestResults(colouredShapePipelineResult);
}
@@ -43,7 +44,8 @@ public class ColoredShapePipelineTest {
pipeline.settings = settings;
CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera);
TestUtils.showImage(
colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Quadrilateral.");
colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(),
"Pipeline output: Quadrilateral.");
printTestResults(colouredShapePipelineResult);
}
@@ -53,7 +55,8 @@ public class ColoredShapePipelineTest {
pipeline.settings = settings;
CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera);
TestUtils.showImage(
colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Custom.");
colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(),
"Pipeline output: Custom.");
printTestResults(colouredShapePipelineResult);
}
@@ -64,7 +67,8 @@ public class ColoredShapePipelineTest {
pipeline.settings = settings;
CVPipelineResult colouredShapePipelineResult = pipeline.run(frame, QuirkyCamera.DefaultCamera);
TestUtils.showImage(
colouredShapePipelineResult.outputFrame.image.getMat(), "Pipeline output: Circle.");
colouredShapePipelineResult.inputAndOutputFrame.processedImage.getMat(),
"Pipeline output: Circle.");
printTestResults(colouredShapePipelineResult);
}

16
photon-core/src/test/java/org/photonvision/vision/pipeline/ReflectivePipelineTest.java
View File

@@ -26,6 +26,7 @@ import org.photonvision.vision.frame.provider.FileFrameProvider;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.opencv.ContourGroupingMode;
import org.photonvision.vision.opencv.ContourIntersectionDirection;
import org.photonvision.vision.pipe.impl.HSVPipe;
import org.photonvision.vision.pipeline.result.CVPipelineResult;
public class ReflectivePipelineTest {
@@ -45,8 +46,16 @@ public class ReflectivePipelineTest {
new FileFrameProvider(
TestUtils.getWPIImagePath(TestUtils.WPI2019Image.kCargoStraightDark72in_HighRes, false),
TestUtils.WPI2019Image.FOV);
frameProvider.requestFrameThresholdType(pipeline.getThresholdType());
var hsvParams =
new HSVPipe.HSVParams(
pipeline.getSettings().hsvHue,
pipeline.getSettings().hsvSaturation,
pipeline.getSettings().hsvValue,
pipeline.getSettings().hueInverted);
frameProvider.requestHsvSettings(hsvParams);
TestUtils.showImage(frameProvider.get().image.getMat(), "Pipeline input", 1);
TestUtils.showImage(frameProvider.get().colorImage.getMat(), "Pipeline input", 1);
CVPipelineResult pipelineResult;
@@ -56,7 +65,7 @@ public class ReflectivePipelineTest {
Assertions.assertTrue(pipelineResult.hasTargets());
Assertions.assertEquals(2, pipelineResult.targets.size(), "Target count wrong!");
TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output");
TestUtils.showImage(pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output");
}
@Test
@@ -77,7 +86,8 @@ public class ReflectivePipelineTest {
CVPipelineResult pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera);
printTestResults(pipelineResult);
TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output");
TestUtils.showImage(
pipelineResult.inputAndOutputFrame.processedImage.getMat(), "Pipeline output");
}
private static void continuouslyRunPipeline(Frame frame, ReflectivePipelineSettings settings) {

30
photon-core/src/test/java/org/photonvision/vision/pipeline/SolvePNPTest.java
View File

@@ -34,6 +34,7 @@ import org.photonvision.vision.frame.provider.FileFrameProvider;
import org.photonvision.vision.opencv.CVMat;
import org.photonvision.vision.opencv.ContourGroupingMode;
import org.photonvision.vision.opencv.ContourIntersectionDirection;
import org.photonvision.vision.pipe.impl.HSVPipe;
import org.photonvision.vision.pipeline.result.CVPipelineResult;
import org.photonvision.vision.target.TargetModel;
import org.photonvision.vision.target.TrackedTarget;
@@ -105,6 +106,15 @@ public class SolvePNPTest {
TestUtils.WPI2019Image.FOV,
TestUtils.get2019LifeCamCoeffs(false));
frameProvider.requestFrameThresholdType(pipeline.getThresholdType());
var hsvParams =
new HSVPipe.HSVParams(
pipeline.getSettings().hsvHue,
pipeline.getSettings().hsvSaturation,
pipeline.getSettings().hsvValue,
pipeline.getSettings().hueInverted);
frameProvider.requestHsvSettings(hsvParams);
CVPipelineResult pipelineResult;
pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera);
@@ -125,7 +135,8 @@ public class SolvePNPTest {
Assertions.assertEquals(
1, new Translation3d(0, 0, 1).rotateBy(pose.getRotation()).getZ(), 0.05);
TestUtils.showImage(pipelineResult.outputFrame.image.getMat(), "Pipeline output", 999999);
TestUtils.showImage(
pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999);
}
@Test
@@ -147,16 +158,22 @@ public class SolvePNPTest {
TestUtils.WPI2020Image.FOV,
TestUtils.get2020LifeCamCoeffs(false));
frameProvider.requestFrameThresholdType(pipeline.getThresholdType());
var hsvParams =
new HSVPipe.HSVParams(
pipeline.getSettings().hsvHue,
pipeline.getSettings().hsvSaturation,
pipeline.getSettings().hsvValue,
pipeline.getSettings().hueInverted);
frameProvider.requestHsvSettings(hsvParams);
CVPipelineResult pipelineResult = pipeline.run(frameProvider.get(), QuirkyCamera.DefaultCamera);
printTestResults(pipelineResult);
// Draw on input
var outputPipe = new OutputStreamPipeline();
outputPipe.process(
pipelineResult.inputFrame,
pipelineResult.outputFrame,
pipeline.getSettings(),
pipelineResult.targets);
pipelineResult.inputAndOutputFrame, pipeline.getSettings(), pipelineResult.targets);
// these numbers are not *accurate*, but they are known and expected
var pose = pipelineResult.targets.get(0).getBestCameraToTarget3d();
@@ -165,7 +182,8 @@ public class SolvePNPTest {
// Z rotation should be mostly facing us
Assertions.assertEquals(Units.degreesToRadians(-140), pose.getRotation().getZ(), 1);
TestUtils.showImage(pipelineResult.inputFrame.image.getMat(), "Pipeline output", 999999);
TestUtils.showImage(
pipelineResult.inputAndOutputFrame.colorImage.getMat(), "Pipeline output", 999999);
}
private static void continuouslyRunPipeline(Frame frame, ReflectivePipelineSettings settings) {

11
photon-server/src/main/java/org/photonvision/Main.java
View File

@@ -37,6 +37,7 @@ import org.photonvision.common.logging.Logger;
import org.photonvision.common.networking.NetworkManager;
import org.photonvision.common.util.TestUtils;
import org.photonvision.common.util.numbers.IntegerCouple;
import org.photonvision.raspi.LibCameraJNI;
import org.photonvision.server.Server;
import org.photonvision.vision.camera.FileVisionSource;
import org.photonvision.vision.opencv.CVMat;
@@ -293,11 +294,11 @@ public class Main {
logger.error("Failed to load native libraries!", e);
}
// try {
// PicamJNI.forceLoad();
// } catch (IOException e) {
// logger.error("Failed to load Picam JNI!", e);
// }
try {
LibCameraJNI.forceLoad();
} catch (IOException e) {
logger.error("Failed to load native libraries!", e);
}
try {
if (!handleArgs(args)) {

BIN
photon-server/src/main/resources/nativelibraries/libphotonlibcamera.so Executable file
View File

Binary file not shown.

18
scripts/generatePiImage.sh
View File

@@ -16,8 +16,24 @@ LOOP=$(sudo losetup --show -fP "${IMAGE_FILE}")
sudo mount ${LOOP}p2 $TMP
pushd .
cd $TMP/opt/photonvision
ls
sudo cp $NEW_JAR photonvision.jar
cd $TMP/etc/systemd/system/multi-user.target.wants
sudo bash -c "printf \
\"[Unit]
Description=Service that runs PhotonVision
[Service]
WorkingDirectory=/opt/photonvision
ExecStart=/usr/bin/java -Xmx512m -jar /opt/photonvision/photonvision.jar
ExecStop=/bin/systemctl kill photonvision
Type=simple
Restart=on-failure
RestartSec=1
[Install]
WantedBy=multi-user.target\" > photonvision.service"
popd
sudo umount ${TMP}
sudo rmdir ${TMP}

2
scripts/install.sh
View File

@@ -68,7 +68,7 @@ Nice=-10
# look up the right values for your CPU
# AllowCPUs=4-7
ExecStart=/usr/bin/java -jar /opt/photonvision/photonvision.jar -Xmx512m
ExecStart=/usr/bin/java -Xmx512m -jar /opt/photonvision/photonvision.jar
ExecStop=/bin/systemctl kill photonvision
Type=simple
Restart=on-failure