/* * MIT License * * Copyright (c) PhotonVision * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ package org.photonvision.simulation; import edu.wpi.first.cameraserver.CameraServer; import edu.wpi.first.cscore.CvSource; import edu.wpi.first.cscore.VideoMode.PixelFormat; import edu.wpi.first.cscore.VideoSource.ConnectionStrategy; import edu.wpi.first.math.MathUtil; import edu.wpi.first.math.Pair; import edu.wpi.first.math.geometry.Pose3d; import edu.wpi.first.util.RuntimeLoader; import edu.wpi.first.util.WPIUtilJNI; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.opencv.core.Core; import org.opencv.core.CvType; import org.opencv.core.Mat; import org.opencv.core.Point; import org.opencv.core.Size; import org.opencv.imgproc.Imgproc; import org.photonvision.PhotonCamera; import org.photonvision.PhotonTargetSortMode; import org.photonvision.common.dataflow.structures.Packet; import org.photonvision.common.networktables.NTTopicSet; import org.photonvision.estimation.CameraTargetRelation; import org.photonvision.estimation.OpenCVHelp; import org.photonvision.estimation.PNPResults; import org.photonvision.targeting.PhotonPipelineResult; import org.photonvision.targeting.PhotonTrackedTarget; import org.photonvision.targeting.TargetCorner; /** * A handle for simulating {@link PhotonCamera} values. Processing simulated targets through this * class will change the associated PhotonCamera's results. */ @SuppressWarnings("unused") public class PhotonCameraSim implements AutoCloseable { private final PhotonCamera cam; NTTopicSet ts = new NTTopicSet(); private long heartbeatCounter = 0; /** This simulated camera's {@link SimCameraProperties} */ public final SimCameraProperties prop; private long nextNTEntryTime = WPIUtilJNI.now(); private double maxSightRangeMeters = Double.MAX_VALUE; private static final double kDefaultMinAreaPx = 90; private double minTargetAreaPercent; private PhotonTargetSortMode sortMode = PhotonTargetSortMode.Largest; // video stream simulation private final CvSource videoSimRaw; private final Mat videoSimFrameRaw = new Mat(); private boolean videoSimRawEnabled = true; private final CvSource videoSimProcessed; private final Mat videoSimFrameProcessed = new Mat(); private boolean videoSimProcEnabled = true; static { try { var loader = new RuntimeLoader<>( Core.NATIVE_LIBRARY_NAME, RuntimeLoader.getDefaultExtractionRoot(), Core.class); loader.loadLibrary(); } catch (Exception e) { throw new RuntimeException("Failed to load native libraries!", e); } } @Override public void close() { videoSimRaw.close(); videoSimFrameRaw.release(); videoSimProcessed.close(); videoSimFrameProcessed.release(); } /** * Constructs a handle for simulating {@link PhotonCamera} values. Processing simulated targets * through this class will change the associated PhotonCamera's results. * *
This constructor's camera has a 90 deg FOV with no simulated lag! * *
By default, the minimum target area is 100 pixels and there is no maximum sight range. * * @param camera The camera to be simulated */ public PhotonCameraSim(PhotonCamera camera) { this(camera, SimCameraProperties.PERFECT_90DEG()); } /** * Constructs a handle for simulating {@link PhotonCamera} values. Processing simulated targets * through this class will change the associated PhotonCamera's results. * *
By default, the minimum target area is 100 pixels and there is no maximum sight range.
*
* @param camera The camera to be simulated
* @param prop Properties of this camera such as FOV and FPS
*/
public PhotonCameraSim(PhotonCamera camera, SimCameraProperties prop) {
this.cam = camera;
this.prop = prop;
setMinTargetAreaPixels(kDefaultMinAreaPx);
videoSimRaw =
CameraServer.putVideo(camera.getName() + "-raw", prop.getResWidth(), prop.getResHeight());
videoSimRaw.setPixelFormat(PixelFormat.kGray);
videoSimProcessed =
CameraServer.putVideo(
camera.getName() + "-processed", prop.getResWidth(), prop.getResHeight());
ts.removeEntries();
ts.subTable = camera.getCameraTable();
ts.updateEntries();
}
/**
* Constructs a handle for simulating {@link PhotonCamera} values. Processing simulated targets
* through this class will change the associated PhotonCamera's results.
*
* @param camera The camera to be simulated
* @param prop Properties of this camera such as FOV and FPS
* @param minTargetAreaPercent The minimum percentage(0 - 100) a detected target must take up of
* the camera's image to be processed. Match this with your contour filtering settings in the
* PhotonVision GUI.
* @param maxSightRangeMeters Maximum distance at which the target is illuminated to your camera.
* Note that minimum target area of the image is separate from this.
*/
public PhotonCameraSim(
PhotonCamera camera,
SimCameraProperties prop,
double minTargetAreaPercent,
double maxSightRangeMeters) {
this(camera, prop);
this.minTargetAreaPercent = minTargetAreaPercent;
this.maxSightRangeMeters = maxSightRangeMeters;
}
public PhotonCamera getCamera() {
return cam;
}
public double getMinTargetAreaPercent() {
return minTargetAreaPercent;
}
public double getMinTargetAreaPixels() {
return minTargetAreaPercent / 100.0 * prop.getResArea();
}
public double getMaxSightRangeMeters() {
return maxSightRangeMeters;
}
public PhotonTargetSortMode getTargetSortMode() {
return sortMode;
}
public CvSource getVideoSimRaw() {
return videoSimRaw;
}
public Mat getVideoSimFrameRaw() {
return videoSimFrameRaw;
}
/**
* Determines if this target's pose should be visible to the camera without considering its
* projected image points. Does not account for image area.
*
* @param camPose Camera's 3d pose
* @param target Vision target containing pose and shape
* @return If this vision target can be seen before image projection.
*/
public boolean canSeeTargetPose(Pose3d camPose, VisionTargetSim target) {
var rel = new CameraTargetRelation(camPose, target.getPose());
return (
// target translation is outside of camera's FOV
(Math.abs(rel.camToTargYaw.getDegrees()) < prop.getHorizFOV().getDegrees() / 2)
&& (Math.abs(rel.camToTargPitch.getDegrees()) < prop.getVertFOV().getDegrees() / 2)
&& (!target.getModel().isPlanar
|| Math.abs(rel.targToCamAngle.getDegrees())
< 90) // camera is behind planar target and it should be occluded
&& (rel.camToTarg.getTranslation().getNorm() <= maxSightRangeMeters)); // target is too far
}
/**
* Determines if all target corners are inside the camera's image.
*
* @param corners The corners of the target as image points(x,y)
*/
public boolean canSeeCorners(List