/* * MIT License * * Copyright (c) PhotonVision * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #pragma once #include #include #include #include #include #include #include #include #include #include namespace photon { /** * Calibration and performance values for this camera. * * The resolution will affect the accuracy of projected(3d to 2d) target * corners and similarly the severity of image noise on estimation(2d to 3d). * * The camera intrinsics and distortion coefficients describe the results of * calibration, and how to map between 3d field points and 2d image points. * * The performance values (framerate/exposure time, latency) determine how * often results should be updated and with how much latency in simulation. High * exposure time causes motion blur which can inhibit target detection while * moving. Note that latency estimation does not account for network latency and * the latency reported will always be perfect. */ class SimCameraProperties { public: /** Default constructor which is the same as PERFECT_90DEG */ SimCameraProperties() { SetCalibration(960, 720, frc::Rotation2d{90_deg}); } /** * Reads camera properties from a PhotonVision config.json file. * This is only the resolution, camera intrinsics, distortion coefficients, * and average/std. dev. pixel error. Other camera properties must be set. * * @param path Path to the config.json file * @param width The width of the desired resolution in the JSON * @param height The height of the desired resolution in the JSON */ SimCameraProperties(std::string path, int width, int height) {} void SetCalibration(int width, int height, frc::Rotation2d fovDiag); void SetCalibration(int width, int height, const Eigen::Matrix& newCamIntrinsics, const Eigen::Matrix& newDistCoeffs); void SetCalibError(double newAvgErrorPx, double newErrorStdDevPx) { avgErrorPx = newAvgErrorPx; errorStdDevPx = newErrorStdDevPx; } /** * Sets the simulated FPS for this camera. * * @param fps The average frames per second the camera should process at. * **Exposure time limits FPS if set!** */ void SetFPS(units::hertz_t fps) { frameSpeed = units::math::max(1 / fps, exposureTime); } /** * Sets the simulated exposure time for this camera. * * @param exposureTime The amount of time the "shutter" is open for one frame. * Affects motion blur. **Frame speed(from FPS) is limited to this!** */ void SetExposureTime(units::second_t exposureTime) { this->exposureTime = exposureTime; frameSpeed = units::math::max(frameSpeed, this->exposureTime); } /** * Sets the simulated latency for this camera. * * @param avgLatency The average latency (from image capture to data * published) a frame should have */ void SetAvgLatency(units::second_t avgLatency) { this->avgLatency = avgLatency; } /** * Sets the simulated latency variation for this camera. * * @param latencyStdDev The standard deviation of the latency */ void SetLatencyStdDev(units::second_t latencyStdDev) { this->latencyStdDev = latencyStdDev; } /** * Gets the width of the simulated camera image. * * @return The width in pixels */ int GetResWidth() const { return resWidth; } /** * Gets the height of the simulated camera image. * * @return The height in pixels */ int GetResHeight() const { return resHeight; } /** * Gets the area of the simulated camera image. * * @return The area in pixels */ int GetResArea() const { return resWidth * resHeight; } double GetAspectRatio() const { return static_cast(resWidth) / static_cast(resHeight); } Eigen::Matrix GetIntrinsics() const { return camIntrinsics; } /** * Returns the camera calibration's distortion coefficients, in OPENCV8 form. * Higher-order terms are set to 0 * * @return The distortion coefficients in an 8x1 matrix */ Eigen::Matrix GetDistCoeffs() const { return distCoeffs; } /** * Gets the FPS of the simulated camera. * * @return The FPS */ units::hertz_t GetFPS() const { return 1 / frameSpeed; } /** * Gets the time per frame of the simulated camera. * * @return The time per frame */ units::second_t GetFrameSpeed() const { return frameSpeed; } /** * Gets the exposure time of the simulated camera. * * @return The exposure time */ units::second_t GetExposureTime() const { return exposureTime; } /** * Gets the average latency of the simulated camera. * * @return The average latency */ units::second_t GetAverageLatency() const { return avgLatency; } /** * Gets the time per frame of the simulated camera. * * @return The time per frame */ units::second_t GetLatencyStdDev() const { return latencyStdDev; } /** * The percentage (0 - 100) of this camera's resolution the contour takes up * in pixels of the image. * * @param points Points of the contour * @return The percentage */ double GetContourAreaPercent(const std::vector& points) { return cv::contourArea(photon::OpenCVHelp::GetConvexHull(points)) / GetResArea() * 100; } /** The yaw from the principal point of this camera to the pixel x value. * Positive values left. */ frc::Rotation2d GetPixelYaw(double pixelX) const { double fx = camIntrinsics(0, 0); double cx = camIntrinsics(0, 2); double xOffset = cx - pixelX; return frc::Rotation2d{fx, xOffset}; } /** * The pitch from the principal point of this camera to the pixel y value. * Pitch is positive down. * * Note that this angle is naively computed and may be incorrect. See * #getCorrectedPixelRot(const cv::Point2d). */ frc::Rotation2d GetPixelPitch(double pixelY) const { double fy = camIntrinsics(1, 1); double cy = camIntrinsics(1, 2); double yOffset = cy - pixelY; return frc::Rotation2d{fy, -yOffset}; } /** * Finds the yaw and pitch to the given image point. Yaw is positive left, and * pitch is positive down. * * Note that pitch is naively computed and may be incorrect. See * #getCorrectedPixelRot(const cv::Point2d). */ frc::Rotation3d GetPixelRot(const cv::Point2d& point) const { return frc::Rotation3d{0_rad, GetPixelPitch(point.y).Radians(), GetPixelYaw(point.x).Radians()}; } /** * Gives the yaw and pitch of the line intersecting the camera lens and the * given pixel coordinates on the sensor. Yaw is positive left, and pitch * positive down. * * The pitch traditionally calculated from pixel offsets do not correctly * account for non-zero values of yaw because of perspective distortion (not * to be confused with lens distortion)-- for example, the pitch angle is * naively calculated as: * *
pitch = arctan(pixel y offset / focal length y)
   *
   * However, using focal length as a side of the associated right triangle is
   * not correct when the pixel x value is not 0, because the distance from this
   * pixel (projected on the x-axis) to the camera lens increases. Projecting a
   * line back out of the camera with these naive angles will not intersect the
   * 3d point that was originally projected into this 2d pixel. Instead, this
   * length should be:
   *
   * 
focal length y ⟶ (focal length y / cos(arctan(pixel x offset / focal
   * length x)))
* * @return Rotation3d with yaw and pitch of the line projected out of the * camera from the given pixel (roll is zero). */ frc::Rotation3d GetCorrectedPixelRot(const cv::Point2d& point) const { double fx = camIntrinsics(0, 0); double cx = camIntrinsics(0, 2); double xOffset = cx - point.x; double fy = camIntrinsics(1, 1); double cy = camIntrinsics(1, 2); double yOffset = cy - point.y; frc::Rotation2d yaw{fx, xOffset}; frc::Rotation2d pitch{fy / std::cos(std::atan(xOffset / fx)), -yOffset}; return frc::Rotation3d{0_rad, pitch.Radians(), yaw.Radians()}; } frc::Rotation2d GetHorizFOV() const { frc::Rotation2d left = GetPixelYaw(0); frc::Rotation2d right = GetPixelYaw(resWidth); return left - right; } frc::Rotation2d GetVertFOV() const { frc::Rotation2d above = GetPixelPitch(0); frc::Rotation2d below = GetPixelPitch(resHeight); return below - above; } frc::Rotation2d GetDiagFOV() const { return frc::Rotation2d{ units::math::hypot(GetHorizFOV().Radians(), GetVertFOV().Radians())}; } /** * Determines where the line segment defined by the two given translations * intersects the camera's frustum/field-of-vision, if at all. * *

The line is parametrized so any of its points p = t * (b - a) + * a. This method returns these values of t, minimum first, defining * the region of the line segment which is visible in the frustum. If both * ends of the line segment are visible, this simply returns {0, 1}. If, for * example, point b is visible while a is not, and half of the line segment is * inside the camera frustum, {0.5, 1} would be returned. * * @param camRt The change in basis from world coordinates to camera * coordinates. See RotTrlTransform3d#makeRelativeTo(frc::Pose3d). * @param a The initial translation of the line * @param b The final translation of the line * @return A Pair of Doubles. The values may be empty: * - {double, double} : Two parametrized values(t), minimum first, * representing which segment of the line is visible in the camera frustum. * - {double, std::nullopt} : One value(t) representing a single * intersection point. For example, the line only intersects the intersection * of two adjacent viewplanes. * - {std::nullopt, std::nullopt} : No values. The line segment is not * visible in the camera frustum. */ std::pair, std::optional> GetVisibleLine( const RotTrlTransform3d& camRt, const frc::Translation3d& a, const frc::Translation3d& b) const; /** * Returns these points after applying this camera's estimated noise. * * @param points The points to add noise to * @return The points with noise */ std::vector EstPixelNoise( const std::vector& points) { if (avgErrorPx == 0 && errorStdDevPx == 0) { return points; } std::vector noisyPts; noisyPts.reserve(points.size()); for (size_t i = 0; i < points.size(); i++) { cv::Point2f p = points[i]; float error = avgErrorPx + gaussian(generator) * errorStdDevPx; float errorAngle = uniform(generator) * 2 * std::numbers::pi - std::numbers::pi; noisyPts.emplace_back(cv::Point2f{p.x + error * std::cos(errorAngle), p.y + error * std::sin(errorAngle)}); } return noisyPts; } /** * Returns an estimation of a frame's processing latency with noise added. * * @return The latency estimate */ units::second_t EstLatency() { return units::math::max(avgLatency + gaussian(generator) * latencyStdDev, 0_s); } /** * Estimates how long until the next frame should be processed. * * @return The estimated time until the next frame */ units::second_t EstSecUntilNextFrame() { return frameSpeed + units::math::max(0_s, EstLatency() - frameSpeed); } /** * Creates a set of camera properties where the camera has a 960x720 * resolution, 90 degree FOV, and is a "perfect" lagless camera. * * @return The properties for this theoretical camera */ static SimCameraProperties PERFECT_90DEG() { return SimCameraProperties{}; } /** * Creates a set of camera properties matching those of Microsoft Lifecam * running on a Raspberry Pi 4 at 320x240 resolution. * * Note that this set of properties represents *a camera setup*, not *your * camera setup*. Do not use these camera properties for any non-sim vision * calculations, especially the calibration data. Always use your camera's * calibration data to do vision calculations in non-sim environments. These * properties exist as a sample that may be used to get representative data in * sim. * * @return The properties for this camera setup */ static SimCameraProperties PI4_LIFECAM_320_240() { SimCameraProperties prop{}; prop.SetCalibration( 320, 240, (Eigen::MatrixXd(3, 3) << 328.2733242048587, 0.0, 164.8190261141906, 0.0, 318.0609794305216, 123.8633838438093, 0.0, 0.0, 1.0) .finished(), Eigen::Matrix{ 0.09957946553445934, -0.9166265114485799, 0.0019519890627236526, -0.0036071725380870333, 1.5627234622420942, 0, 0, 0}); prop.SetCalibError(0.21, 0.0124); prop.SetFPS(30_Hz); prop.SetAvgLatency(30_ms); prop.SetLatencyStdDev(10_ms); return prop; } /** * Creates a set of camera properties matching those of Microsoft Lifecam * running on a Raspberry Pi 4 at 640x480 resolution. * *

Note that this set of properties represents *a camera setup*, not *your * camera setup*. Do not use these camera properties for any non-sim vision * calculations, especially the calibration data. Always use your camera's * calibration data to do vision calculations in non-sim environments. These * properties exist as a sample that may be used to get representative data in * sim. * * @return The properties for this camera setup */ static SimCameraProperties PI4_LIFECAM_640_480() { SimCameraProperties prop{}; prop.SetCalibration( 640, 480, (Eigen::MatrixXd(3, 3) << 669.1428078983059, 0.0, 322.53377249329213, 0.0, 646.9843137061716, 241.26567383784163, 0.0, 0.0, 1.0) .finished(), Eigen::Matrix{ 0.12788470750464645, -1.2350335805796528, 0.0024990767286192732, -0.0026958287600230705, 2.2951386729115537, 0, 0, 0}); prop.SetCalibError(0.26, 0.046); prop.SetFPS(15_Hz); prop.SetAvgLatency(65_ms); prop.SetLatencyStdDev(15_ms); return prop; } /** * Creates a set of camera properties matching those of a Limelight 2 running * at 640x480 resolution. * *

Note that this set of properties represents *a camera setup*, not *your * camera setup*. Do not use these camera properties for any non-sim vision * calculations, especially the calibration data. Always use your camera's * calibration data to do vision calculations in non-sim environments. These * properties exist as a sample that may be used to get representative data in * sim. * * @return The properties for this camera setup */ static SimCameraProperties LL2_640_480() { SimCameraProperties prop{}; prop.SetCalibration( 640, 480, (Eigen::MatrixXd(3, 3) << 511.22843367007755, 0.0, 323.62049380211096, 0.0, 514.5452336723849, 261.8827920543568, 0.0, 0.0, 1.0) .finished(), Eigen::Matrix{0.1917469998873756, -0.5142936883324216, 0.012461562046896614, 0.0014084973492408186, 0.35160648971214437, 0, 0, 0}); prop.SetCalibError(0.25, 0.05); prop.SetFPS(15_Hz); prop.SetAvgLatency(35_ms); prop.SetLatencyStdDev(8_ms); return prop; } /** * Creates a set of camera properties matching those of a Limelight 2 running * at 960x720 resolution. * *

Note that this set of properties represents *a camera setup*, not *your * camera setup*. Do not use these camera properties for any non-sim vision * calculations, especially the calibration data. Always use your camera's * calibration data to do vision calculations in non-sim environments. These * properties exist as a sample that may be used to get representative data in * sim. * * @return The properties for this camera setup */ static SimCameraProperties LL2_960_720() { SimCameraProperties prop{}; prop.SetCalibration( 960, 720, (Eigen::MatrixXd(3, 3) << 769.6873145148892, 0.0, 486.1096609458122, 0.0, 773.8164483705323, 384.66071662358354, 0.0, 0.0, 1.0) .finished(), Eigen::Matrix{0.189462064814501, -0.49903003669627627, 0.007468423590519429, 0.002496885298683693, 0.3443122090208624, 0, 0, 0}); prop.SetCalibError(0.35, 0.10); prop.SetFPS(10_Hz); prop.SetAvgLatency(50_ms); prop.SetLatencyStdDev(15_ms); return prop; } /** * Creates a set of camera properties matching those of a Limelight 2 running * at 1280x720 resolution. * *

Note that this set of properties represents *a camera setup*, not *your * camera setup*. Do not use these camera properties for any non-sim vision * calculations, especially the calibration data. Always use your camera's * calibration data to do vision calculations in non-sim environments. These * properties exist as a sample that may be used to get representative data in * sim. * * @return The properties for this camera setup */ static SimCameraProperties LL2_1280_720() { SimCameraProperties prop{}; prop.SetCalibration( 1280, 720, (Eigen::MatrixXd(3, 3) << 1011.3749416937393, 0.0, 645.4955139388737, 0.0, 1008.5391755084075, 508.32877656020196, 0.0, 0.0, 1.0) .finished(), Eigen::Matrix{0.13730101577061535, -0.2904345656989261, 8.32475714507539E-4, -3.694397782014239E-4, 0.09487962227027584, 0, 0, 0}); prop.SetCalibError(0.37, 0.06); prop.SetFPS(7_Hz); prop.SetAvgLatency(60_ms); prop.SetLatencyStdDev(20_ms); return prop; } private: std::mt19937 generator{std::random_device{}()}; std::normal_distribution gaussian{0.0, 1.0}; std::uniform_real_distribution uniform{0.0, 1.0}; int resWidth; int resHeight; Eigen::Matrix camIntrinsics; Eigen::Matrix distCoeffs; double avgErrorPx{0}; double errorStdDevPx{0}; units::second_t frameSpeed{0}; units::second_t exposureTime{0}; units::second_t avgLatency{0}; units::second_t latencyStdDev{0}; std::vector> viewplanes{}; }; } // namespace photon