Support more charuco boards (#1348)

Add support for the old opencv charuco board like calibio. 

Add support for other tag families while calibrating.

Fix calibration issue index out of range with charuco missing points.

photon-client/src/components/cameras/CameraCalibrationCard.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { computed, ref } from "vue";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { CalibrationBoardTypes, type VideoFormat } from "@/types/SettingTypes";
+import { CalibrationBoardTypes, CalibrationTagFamilies, type VideoFormat } from "@/types/SettingTypes";
 import JsPDF from "jspdf";
 import { font as PromptRegular } from "@/assets/fonts/PromptRegular";
 import MonoLogo from "@/assets/images/logoMono.png";
@@ -79,6 +79,8 @@ const markerSizeIn = ref(0.75);
 const patternWidth = ref(8);
 const patternHeight = ref(8);
 const boardType = ref<CalibrationBoardTypes>(CalibrationBoardTypes.Charuco);
+const useOldPattern = ref(false);
+const tagFamily = ref<CalibrationTagFamilies>(CalibrationTagFamilies.Dict_4X4_1000);
 const useMrCalRef = ref(true);
 const useMrCal = computed<boolean>({
   get() {
@@ -200,7 +202,9 @@ const startCalibration = () => {
     patternHeight: patternHeight.value,
     patternWidth: patternWidth.value,
     boardType: boardType.value,
-    useMrCal: useMrCal.value
+    useMrCal: useMrCal.value,
+    useOldPattern: useOldPattern.value,
+    tagFamily: tagFamily.value
   });
   // The Start PnP method already handles updating the backend so only a store update is required
   useCameraSettingsStore().currentCameraSettings.currentPipelineIndex = WebsocketPipelineType.Calib3d;
@@ -302,6 +306,15 @@ const setSelectedVideoFormat = (format: VideoFormat) => {
               :items="['Chessboard', 'Charuco']"
               :disabled="isCalibrating"
             />
+            <pv-select
+              v-model="tagFamily"
+              v-show="boardType == CalibrationBoardTypes.Charuco"
+              label="Tag Family"
+              tooltip="Dictionary of aruco markers on the charuco board"
+              :select-cols="7"
+              :items="['Dict_4X4_1000', 'Dict_5X5_1000', 'Dict_6X6_1000', 'Dict_7X7_1000']"
+              :disabled="isCalibrating"
+            />
             <pv-number-input
               v-model="squareSizeIn"
               label="Pattern Spacing (in)"
@@ -335,6 +348,14 @@ const setSelectedVideoFormat = (format: VideoFormat) => {
               :rules="[(v) => v >= 4 || 'Height must be at least 4']"
               :label-cols="5"
             />
+            <pv-switch
+              v-model="useOldPattern"
+              v-show="boardType == CalibrationBoardTypes.Charuco"
+              label="Old OpenCV Pattern"
+              :disabled="isCalibrating"
+              tooltip="If enabled, Photon will use the old OpenCV pattern for calibration."
+              :label-cols="5"
+            />
             <pv-switch
               v-model="useMrCal"
               label="Try using MrCal over OpenCV"

photon-client/src/stores/settings/CameraSettingsStore.ts

@@ -1,5 +1,6 @@
 import { defineStore } from "pinia";
 import type {
+  CalibrationTagFamilies,
   CalibrationBoardTypes,
   CameraCalibrationResult,
   CameraSettings,
@@ -319,6 +320,8 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
         patternHeight: number;
         boardType: CalibrationBoardTypes;
         useMrCal: boolean;
+        useOldPattern: boolean;
+        tagFamily: CalibrationTagFamilies;
       },
       cameraIndex: number = useStateStore().currentCameraIndex
     ) {

photon-client/src/types/SettingTypes.ts

@@ -295,6 +295,13 @@ export enum CalibrationBoardTypes {
   Charuco = 1
 }
 
+export enum CalibrationTagFamilies {
+  Dict_4X4_1000 = 0,
+  Dict_5X5_1000 = 1,
+  Dict_6X6_1000 = 2,
+  Dict_7X7_1000 = 3
+}
+
 export enum RobotOffsetType {
   Clear = 0,
   Single = 1,

photon-core/src/main/java/org/photonvision/vision/pipe/impl/FindBoardCornersPipe.java

@@ -113,7 +113,8 @@ public class FindBoardCornersPipe
                             new Size(params.boardWidth, params.boardHeight),
                             (float) params.gridSize,
                             (float) params.markerSize,
-                            Objdetect.getPredefinedDictionary(params.tagFamily));
+                            Objdetect.getPredefinedDictionary(params.tagFamily.getValue()));
+            board.setLegacyPattern(params.useOldPattern);
             detector = new CharucoDetector(board);
         } else {
             logger.error("Can't create pattern for unknown board type " + params.type);
@@ -309,6 +310,7 @@ public class FindBoardCornersPipe
                 }
 
                 outBoardCorners.fromArray(boardCorners);
+                objPts.fromArray(objectPoints);
                 outLevels.fromArray(levels);
             }
             imgPoints.release();
@@ -371,16 +373,18 @@ public class FindBoardCornersPipe
         final double gridSize;
         final double markerSize;
         final FrameDivisor divisor;
-        final int tagFamily;
+        final UICalibrationData.TagFamily tagFamily;
+        final boolean useOldPattern;
 
         public FindCornersPipeParams(
                 int boardHeight,
                 int boardWidth,
                 UICalibrationData.BoardType type,
-                int tagFamily,
+                UICalibrationData.TagFamily tagFamily,
                 double gridSize,
                 double markerSize,
-                FrameDivisor divisor) {
+                FrameDivisor divisor,
+                boolean useOldPattern) {
             this.boardHeight = boardHeight;
             this.boardWidth = boardWidth;
             this.tagFamily = tagFamily;
@@ -388,6 +392,7 @@ public class FindBoardCornersPipe
             this.gridSize = gridSize; // meter
             this.markerSize = markerSize; // meter
             this.divisor = divisor;
+            this.useOldPattern = useOldPattern;
         }
 
         @Override
@@ -412,6 +417,8 @@ public class FindBoardCornersPipe
             FindCornersPipeParams other = (FindCornersPipeParams) obj;
             if (boardHeight != other.boardHeight) return false;
             if (boardWidth != other.boardWidth) return false;
+            if (tagFamily != other.tagFamily) return false;
+            if (useOldPattern != other.useOldPattern) return false;
             if (type != other.type) return false;
             if (Double.doubleToLongBits(gridSize) != Double.doubleToLongBits(other.gridSize))
                 return false;

photon-core/src/main/java/org/photonvision/vision/pipeline/Calibrate3dPipeline.java

@@ -90,7 +90,8 @@ public class Calibrate3dPipeline
                         settings.tagFamily,
                         settings.gridSize,
                         settings.markerSize,
-                        settings.streamingFrameDivisor);
+                        settings.streamingFrameDivisor,
+                        settings.useOldPattern);
         findBoardCornersPipe.setParams(findCornersPipeParams);
 
         Calibrate3dPipe.CalibratePipeParams calibratePipeParams =
@@ -227,7 +228,9 @@ public class Calibrate3dPipeline
                                 settings.boardWidth,
                                 settings.boardHeight,
                                 settings.boardType,
-                                settings.useMrCal));
+                                settings.useMrCal,
+                                settings.useOldPattern,
+                                settings.tagFamily));
 
         DataChangeService.getInstance()
                 .publishEvent(OutgoingUIEvent.wrappedOf("calibrationData", state));

photon-core/src/main/java/org/photonvision/vision/pipeline/Calibration3dPipelineSettings.java

@@ -19,19 +19,19 @@ package org.photonvision.vision.pipeline;
 
 import edu.wpi.first.math.util.Units;
 import org.opencv.core.Size;
-import org.opencv.objdetect.Objdetect;
 import org.photonvision.vision.frame.FrameDivisor;
 
 public class Calibration3dPipelineSettings extends AdvancedPipelineSettings {
     public int boardHeight = 8;
     public int boardWidth = 8;
     public UICalibrationData.BoardType boardType = UICalibrationData.BoardType.CHESSBOARD;
-    public int tagFamily = Objdetect.DICT_4X4_50;
+    public UICalibrationData.TagFamily tagFamily = UICalibrationData.TagFamily.Dict_4X4_1000;
     public double gridSize = Units.inchesToMeters(1.0);
     public double markerSize = Units.inchesToMeters(0.75);
 
     public Size resolution = new Size(640, 480);
     public boolean useMrCal = true;
+    public boolean useOldPattern = false;
 
     public Calibration3dPipelineSettings() {
         super();

photon-core/src/main/java/org/photonvision/vision/pipeline/UICalibrationData.java

@@ -17,6 +17,8 @@
 
 package org.photonvision.vision.pipeline;
 
+import org.opencv.objdetect.Objdetect;
+
 public class UICalibrationData {
     public int videoModeIndex;
     public int count;
@@ -28,6 +30,8 @@ public class UICalibrationData {
     public BoardType boardType;
     public boolean useMrCal;
     public double markerSizeIn;
+    public boolean useOldPattern;
+    public TagFamily tagFamily;
 
     public UICalibrationData() {}
 
@@ -41,7 +45,9 @@ public class UICalibrationData {
             int patternWidth,
             int patternHeight,
             BoardType boardType,
-            boolean useMrCal) {
+            boolean useMrCal,
+            boolean useOldPattern,
+            TagFamily tagFamily) {
         this.count = count;
         this.minCount = minCount;
         this.videoModeIndex = videoModeIndex;
@@ -52,6 +58,8 @@ public class UICalibrationData {
         this.patternHeight = patternHeight;
         this.boardType = boardType;
         this.useMrCal = useMrCal;
+        this.useOldPattern = useOldPattern;
+        this.tagFamily = tagFamily;
     }
 
     public enum BoardType {
@@ -59,6 +67,25 @@ public class UICalibrationData {
         CHARUCOBOARD,
     }
 
+    public enum TagFamily {
+        Dict_4X4_1000(Objdetect.DICT_4X4_1000),
+        Dict_5X5_1000(Objdetect.DICT_5X5_1000),
+        Dict_6X6_1000(Objdetect.DICT_6X6_1000),
+        Dict_7X7_1000(Objdetect.DICT_7X7_1000);
+
+        private int value;
+
+        // getter method
+        public int getValue() {
+            return this.value;
+        }
+
+        // enum constructor - cannot be public or protected
+        private TagFamily(int value) {
+            this.value = value;
+        }
+    }
+
     @Override
     public String toString() {
         return "UICalibrationData{"
@@ -80,6 +107,10 @@ public class UICalibrationData {
                 + patternHeight
                 + ", boardType="
                 + boardType
+                + ", tagFamily="
+                + tagFamily
+                + ", useOldPattern="
+                + useOldPattern
                 + '}';
     }
 }

photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java

@@ -349,6 +349,8 @@ public class VisionModule {
         settings.boardType = data.boardType;
         settings.useMrCal = data.useMrCal;
         settings.resolution = resolution;
+        settings.useOldPattern = data.useOldPattern;
+        settings.tagFamily = data.tagFamily;
 
         // Disable gain if not applicable
         if (!cameraQuirks.hasQuirk(CameraQuirk.Gain)) {

photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java

@@ -32,7 +32,6 @@ import org.opencv.calib3d.Calib3d;
 import org.opencv.core.Mat;
 import org.opencv.core.Size;
 import org.opencv.imgcodecs.Imgcodecs;
-import org.opencv.objdetect.Objdetect;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.LogLevel;
 import org.photonvision.common.logging.Logger;
@@ -46,6 +45,7 @@ import org.photonvision.vision.frame.FrameStaticProperties;
 import org.photonvision.vision.frame.FrameThresholdType;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.pipeline.UICalibrationData.BoardType;
+import org.photonvision.vision.pipeline.UICalibrationData.TagFamily;
 
 public class Calibrate3dPipeTest {
     @BeforeAll
@@ -67,34 +67,52 @@ public class Calibrate3dPipeTest {
                 "lifecam/2024-01-02_lifecam_480",
                 new Size(640, 480),
                 new Size(11, 11),
-                BoardType.CHESSBOARD),
+                BoardType.CHESSBOARD,
+                false),
         SQUARES_LIFECAM_1280(
                 "lifecam/2024-01-02_lifecam_1280",
                 new Size(1280, 720),
                 new Size(11, 11),
-                BoardType.CHESSBOARD),
-
+                BoardType.CHESSBOARD,
+                false),
         CHARUCO_LIFECAM_480(
                 "lifecam/2024-05-07_lifecam_480",
                 new Size(640, 480),
                 new Size(8, 8),
-                BoardType.CHARUCOBOARD),
+                BoardType.CHARUCOBOARD,
+                false),
         CHARUCO_LIFECAM_1280(
                 "lifecam/2024-05-07_lifecam_1280",
                 new Size(1280, 720),
                 new Size(8, 8),
-                BoardType.CHARUCOBOARD);
+                BoardType.CHARUCOBOARD,
+                false),
+        CHARUCO_OLDPATTERN_LIFECAM_480(
+                "lifecam/2024-06-19_lifecam_480_Old_Pattern",
+                new Size(640, 480),
+                new Size(8, 8),
+                BoardType.CHARUCOBOARD,
+                true),
+        CHARUCO_OLDPATTERN_LIFECAM_1280(
+                "lifecam/2024-06-19_lifecam_1280_Old_Pattern",
+                new Size(1280, 720),
+                new Size(8, 8),
+                BoardType.CHARUCOBOARD,
+                true);
 
         final String path;
         final Size size;
         final Size boardSize;
         final BoardType boardType;
+        final boolean useOldPattern;
 
-        private CalibrationDatasets(String path, Size image, Size chessboard, BoardType boardType) {
+        private CalibrationDatasets(
+                String path, Size image, Size chessboard, BoardType boardType, boolean useOldPattern) {
             this.path = path;
             this.size = image;
             this.boardSize = chessboard;
             this.boardType = boardType;
+            this.useOldPattern = useOldPattern;
         }
     }
 
@@ -116,13 +134,30 @@ public class Calibrate3dPipeTest {
         File charucoDir = Path.of(charucoBase, dataset.path).toFile();
 
         if (dataset.boardType == BoardType.CHESSBOARD)
-            calibrateCommon(dataset.size, squareDir, dataset.boardSize, dataset.boardType, useMrCal);
-        else if (dataset.boardType == BoardType.CHESSBOARD)
-            calibrateCommon(dataset.size, charucoDir, dataset.boardSize, dataset.boardType, useMrCal);
+            calibrateCommon(
+                    dataset.size,
+                    squareDir,
+                    dataset.boardSize,
+                    dataset.boardType,
+                    useMrCal,
+                    dataset.useOldPattern);
+        else if (dataset.boardType == BoardType.CHARUCOBOARD)
+            calibrateCommon(
+                    dataset.size,
+                    charucoDir,
+                    dataset.boardSize,
+                    dataset.boardType,
+                    useMrCal,
+                    dataset.useOldPattern);
     }
 
     public static void calibrateCommon(
-            Size imgRes, File rootFolder, Size boardDim, BoardType boardType, boolean useMrCal) {
+            Size imgRes,
+            File rootFolder,
+            Size boardDim,
+            BoardType boardType,
+            boolean useMrCal,
+            boolean useOldPattern) {
         calibrateCommon(
                 imgRes,
                 rootFolder,
@@ -130,10 +165,11 @@ public class Calibrate3dPipeTest {
                 Units.inchesToMeters(1),
                 Units.inchesToMeters(0.75),
                 boardType,
-                Objdetect.DICT_4X4_50,
+                TagFamily.Dict_4X4_1000,
                 imgRes.width / 2,
                 imgRes.height / 2,
-                useMrCal);
+                useMrCal,
+                useOldPattern);
     }
 
     public static void calibrateCommon(
@@ -142,10 +178,11 @@ public class Calibrate3dPipeTest {
             Size boardDim,
             double markerSize,
             BoardType boardType,
-            int tagFamily,
+            TagFamily tagFamily,
             double expectedXCenter,
             double expectedYCenter,
-            boolean useMrCal) {
+            boolean useMrCal,
+            boolean useOldPattern) {
         calibrateCommon(
                 imgRes,
                 rootFolder,
@@ -156,7 +193,8 @@ public class Calibrate3dPipeTest {
                 tagFamily,
                 expectedXCenter,
                 expectedYCenter,
-                useMrCal);
+                useMrCal,
+                useOldPattern);
     }
 
     public static void calibrateCommon(
@@ -166,10 +204,11 @@ public class Calibrate3dPipeTest {
             double boardGridSize_m,
             double markerSize,
             BoardType boardType,
-            int tagFamily,
+            TagFamily tagFamily,
             double expectedXCenter,
             double expectedYCenter,
-            boolean useMrCal) {
+            boolean useMrCal,
+            boolean useOldPattern) {
         int startMatCount = CVMat.getMatCount();
 
         File[] directoryListing = rootFolder.listFiles();
@@ -187,6 +226,7 @@ public class Calibrate3dPipeTest {
         calibration3dPipeline.getSettings().gridSize = boardGridSize_m;
         calibration3dPipeline.getSettings().streamingFrameDivisor = FrameDivisor.NONE;
         calibration3dPipeline.getSettings().useMrCal = useMrCal;
+        calibration3dPipeline.getSettings().useOldPattern = useOldPattern;
 
         for (var file : directoryListing) {
             if (file.isFile()) {