Re-enable OpenCV ArUco detector for AprilTags (#916)

- Fixes ArUco on picam - Adds `ArucoPoseEstimatorPipe` for single-tag pose estimation - Previously, `Aruco.estimatePoseSingleMarkers()` was used for tag pose estimation. This uses the default `SOLVEPNP_ITERATIVE` solver and I believe the method is removed in opencv 4.8. The `SOLVEPNP_IPPE_SQUARE` solver implemented is more appropriate for markers. - Pipeline architecture cleanup - Re-enables ArUco pipeline in UI - Multi-tag support ArUco detector support is still considered experimental at this time. This should enable a baseline of support for initial testing, but expect some quirks to remain across platforms.
2026-06-22 01:11:40 +00:00 · 2023-10-24 19:39:38 -07:00
parent c5b42a1191
commit df45bc2d73
20 changed files with 746 additions and 394 deletions
--- a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue
+++ b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue
@@ -151,8 +151,8 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
  const pipelineTypes = [
    { name: "Reflective", value: WebsocketPipelineType.Reflective },
    { name: "Colored Shape", value: WebsocketPipelineType.ColoredShape },
-    { name: "AprilTag", value: WebsocketPipelineType.AprilTag }
-    // { name: "Aruco", value: WebsocketPipelineType.Aruco }
+    { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
+    { name: "Aruco", value: WebsocketPipelineType.Aruco }
  ];

  if (useCameraSettingsStore().isDriverMode) {
@@ -353,8 +353,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
            :items="[
              { name: 'Reflective', value: WebsocketPipelineType.Reflective },
              { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
-              { name: 'AprilTag', value: WebsocketPipelineType.AprilTag }
-              // { name: 'Aruco', value: WebsocketPipelineType.Aruco }
+              { name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
+              { name: 'Aruco', value: WebsocketPipelineType.Aruco }
            ]"
          />
        </v-card-text>
--- a/photon-client/src/components/dashboard/tabs/ArucoTab.vue
+++ b/photon-client/src/components/dashboard/tabs/ArucoTab.vue
@@ -2,6 +2,9 @@
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { PipelineType } from "@/types/PipelineTypes";
 import PvSlider from "@/components/common/pv-slider.vue";
+import PvSwitch from "@/components/common/pv-switch.vue";
+import PvRangeSlider from "@/components/common/pv-range-slider.vue";
+import PvSelect from "@/components/common/pv-select.vue";
 import { computed, getCurrentInstance } from "vue";
 import { useStateStore } from "@/stores/StateStore";

@@ -20,37 +23,60 @@ const interactiveCols = computed(

 <template>
  <div v-if="currentPipelineSettings.pipelineType === PipelineType.Aruco">
-    <pv-slider
-      v-model="currentPipelineSettings.decimate"
+    <pv-select
+      v-model="currentPipelineSettings.tagFamily"
+      label="Target family"
+      :items="['AprilTag Family 36h11', 'AprilTag Family 25h9', 'AprilTag Family 16h5']"
+      :select-cols="interactiveCols"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ tagFamily: value }, false)"
+    />
+    <pv-switch
+      v-model="currentPipelineSettings.useCornerRefinement"
      class="pt-2"
+      label="Refine Corners"
+      tooltip="Further refine the initial corners with subpixel accuracy."
+      :switch-cols="interactiveCols"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ useCornerRefinement: value }, false)"
+    />
+    <pv-range-slider
+      v-model="currentPipelineSettings.threshWinSizes"
+      label="Thresh Min/Max Size"
+      tooltip="The minimum and maximum adaptive threshold window size. Larger windows tend more towards global thresholding, but small windows can be weak to noise."
+      :min="3"
+      :max="255"
      :slider-cols="interactiveCols"
-      label="Decimate"
-      tooltip="Increases FPS at the expense of range by reducing image resolution initially"
-      :min="1"
-      :max="8"
-      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ decimate: value }, false)"
+      :step="2"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ threshWinSizes: value }, false)"
    />
    <pv-slider
-      v-model="currentPipelineSettings.numIterations"
+      v-model="currentPipelineSettings.threshStepSize"
      class="pt-2"
      :slider-cols="interactiveCols"
-      label="Corner Iterations"
-      tooltip="How many iterations are going to be used in order to refine corners. Higher values are lead to more accuracy at the cost of performance"
-      :min="30"
-      :max="1000"
-      :step="5"
-      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ numIterations: value }, false)"
+      label="Thresh Step Size"
+      tooltip="Smaller values will cause more steps between the min/max sizes. More, varied steps can improve detection robustness to lighting, but may decrease performance."
+      :min="2"
+      :max="128"
+      :step="1"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ threshStepSize: value }, false)"
    />
    <pv-slider
-      v-model="currentPipelineSettings.cornerAccuracy"
+      v-model="currentPipelineSettings.threshConstant"
      class="pt-2"
      :slider-cols="interactiveCols"
-      label="Corner Accuracy"
-      tooltip="Minimum accuracy for the corners, lower is better but more performance intensive "
-      :min="0.01"
-      :max="100"
-      :step="0.01"
-      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ cornerAccuracy: value }, false)"
+      label="Thresh Constant"
+      tooltip="Affects the threshold window mean value cutoff for all steps. Higher values can improve performance, but may harm detection rate."
+      :min="0"
+      :max="128"
+      :step="1"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ threshConstant: value }, false)"
+    />
+    <pv-switch
+      v-model="currentPipelineSettings.debugThreshold"
+      class="pt-2"
+      label="Debug Threshold"
+      tooltip="Display the first threshold step to the color stream."
+      :switch-cols="interactiveCols"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ debugThreshold: value }, false)"
    />
  </div>
 </template>
--- a/photon-client/src/components/dashboard/tabs/OutputTab.vue
+++ b/photon-client/src/components/dashboard/tabs/OutputTab.vue
@@ -86,7 +86,8 @@ const interactiveCols = computed(
    />
    <pv-switch
      v-if="
-        currentPipelineSettings.pipelineType === PipelineType.AprilTag &&
+        (currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
+          currentPipelineSettings.pipelineType === PipelineType.Aruco) &&
        useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
        useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
      "
@@ -99,7 +100,8 @@ const interactiveCols = computed(
    />
    <pv-switch
      v-if="
-        currentPipelineSettings.pipelineType === PipelineType.AprilTag &&
+        (currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
+          currentPipelineSettings.pipelineType === PipelineType.Aruco) &&
        useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
        useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
      "
--- a/photon-client/src/components/dashboard/tabs/TargetsTab.vue
+++ b/photon-client/src/components/dashboard/tabs/TargetsTab.vue
@@ -36,7 +36,8 @@ const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings
              </template>
              <template
                v-if="
-                  useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag &&
+                  (useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+                    useCameraSettingsStore().currentPipelineType === PipelineType.Aruco) &&
                  useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
                "
              >
@@ -67,7 +68,8 @@ const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings
              </template>
              <template
                v-if="
-                  useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag &&
+                  (useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+                    useCameraSettingsStore().currentPipelineType === PipelineType.Aruco) &&
                  useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
                "
              >
@@ -80,7 +82,8 @@ const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings
    </v-row>
    <v-row
      v-if="
-        currentPipelineSettings.pipelineType === PipelineType.AprilTag &&
+        (useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+          useCameraSettingsStore().currentPipelineType === PipelineType.Aruco) &&
        currentPipelineSettings.doMultiTarget &&
        useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
        useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
--- a/photon-client/src/types/PipelineTypes.ts
+++ b/photon-client/src/types/PipelineTypes.ts
@@ -245,11 +245,22 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = {

 export interface ArucoPipelineSettings extends PipelineSettings {
  pipelineType: PipelineType.Aruco;
-  decimate: number;
-  threads: number;
-  numIterations: number;
-  cornerAccuracy: number;
+
+  tagFamily: AprilTagFamily;
+
+  threshWinSizes: WebsocketNumberPair | [number, number];
+  threshStepSize: number;
+  threshConstant: number;
+  debugThreshold: boolean;
+
+  useCornerRefinement: boolean;
+
  useAruco3: boolean;
+  aruco3MinMarkerSideRatio: number;
+  aruco3MinCanonicalImgSide: number;
+
+  doMultiTarget: boolean;
+  doSingleTargetAlways: boolean;
 }
 export type ConfigurableArucoPipelineSettings = Partial<Omit<ArucoPipelineSettings, "pipelineType">> &
  ConfigurablePipelineSettings;
@@ -262,11 +273,17 @@ export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
  ledMode: false,
  pipelineType: PipelineType.Aruco,

-  decimate: 1,
-  threads: 2,
-  numIterations: 100,
-  cornerAccuracy: 25,
-  useAruco3: true
+  tagFamily: AprilTagFamily.Family16h5,
+  threshWinSizes: { first: 11, second: 91 },
+  threshStepSize: 40,
+  threshConstant: 10,
+  debugThreshold: false,
+  useCornerRefinement: true,
+  useAruco3: false,
+  aruco3MinMarkerSideRatio: 0.02,
+  aruco3MinCanonicalImgSide: 32,
+  doMultiTarget: false,
+  doSingleTargetAlways: false
 };

 export type ActivePipelineSettings =