Add RKNN / Object Detection Pipeline (#1144)

Tested on Orange Pi 5 and Cool Pi 4B. Merge with parts of the OpenCV DNN PR. Adds support for YOLOv5s models for Rockchip CPUs with a NPU. Right now hard coded to a note model from alex_idk. Very much still incubating and largely untested.
2026-06-24 01:31:44 +00:00 · 2024-01-15 22:28:34 -05:00
parent e1f550a751
commit 7b67f6bebf
42 changed files with 830 additions and 63 deletions
--- a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue
+++ b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue
@@ -7,6 +7,7 @@ import { computed, ref } from "vue";
 import PvIcon from "@/components/common/pv-icon.vue";
 import PvInput from "@/components/common/pv-input.vue";
 import { PipelineType } from "@/types/PipelineTypes";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";

 const changeCurrentCameraIndex = (index: number) => {
  useCameraSettingsStore().setCurrentCameraIndex(index, true);
@@ -24,6 +25,8 @@ const changeCurrentCameraIndex = (index: number) => {
    case PipelineType.Aruco:
      pipelineType.value = WebsocketPipelineType.Aruco;
      break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
  }
 };

@@ -154,6 +157,9 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
    { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
    { name: "Aruco", value: WebsocketPipelineType.Aruco }
  ];
+  if (useSettingsStore().general.rknnSupported) {
+    pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
+  }

  if (useCameraSettingsStore().isDriverMode) {
    pipelineTypes.push({ name: "Driver Mode", value: WebsocketPipelineType.DriverMode });
@@ -208,6 +214,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
    case PipelineType.Aruco:
      pipelineType.value = WebsocketPipelineType.Aruco;
      break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
+      break;
  }
 });
 </script>
@@ -354,7 +363,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
              { name: 'Reflective', value: WebsocketPipelineType.Reflective },
              { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
              { name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
-              { name: 'Aruco', value: WebsocketPipelineType.Aruco }
+              { name: 'Aruco', value: WebsocketPipelineType.Aruco },
+              { name: 'Object Detection', value: WebsocketPipelineType.ObjectDetection }
            ]"
          />
        </v-card-text>
--- a/photon-client/src/components/dashboard/ConfigOptions.vue
+++ b/photon-client/src/components/dashboard/ConfigOptions.vue
@@ -8,6 +8,7 @@ import ThresholdTab from "@/components/dashboard/tabs/ThresholdTab.vue";
 import ContoursTab from "@/components/dashboard/tabs/ContoursTab.vue";
 import AprilTagTab from "@/components/dashboard/tabs/AprilTagTab.vue";
 import ArucoTab from "@/components/dashboard/tabs/ArucoTab.vue";
+import ObjectDetectionTab from "@/components/dashboard/tabs/ObjectDetectionTab.vue";
 import OutputTab from "@/components/dashboard/tabs/OutputTab.vue";
 import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue";
 import PnPTab from "@/components/dashboard/tabs/PnPTab.vue";
@@ -40,6 +41,10 @@ const allTabs = Object.freeze({
    tabName: "Aruco",
    component: ArucoTab
  },
+  objectDetectionTab: {
+    tabName: "Object Detection",
+    component: ObjectDetectionTab
+  },
  outputTab: {
    tabName: "Output",
    component: OutputTab
@@ -75,6 +80,7 @@ const getTabGroups = (): ConfigOption[][] => {
        allTabs.contoursTab,
        allTabs.apriltagTab,
        allTabs.arucoTab,
+        allTabs.objectDetectionTab,
        allTabs.outputTab
      ],
      [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
@@ -82,14 +88,21 @@ const getTabGroups = (): ConfigOption[][] => {
  } else if (lgAndDown) {
    return [
      [allTabs.inputTab],
-      [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [
+        allTabs.thresholdTab,
+        allTabs.contoursTab,
+        allTabs.apriltagTab,
+        allTabs.arucoTab,
+        allTabs.objectDetectionTab,
+        allTabs.outputTab
+      ],
      [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
    ];
  } else if (xl) {
    return [
      [allTabs.inputTab],
      [allTabs.thresholdTab],
-      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.objectDetectionTab, allTabs.outputTab],
      [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
    ];
  }
@@ -103,17 +116,20 @@ const tabGroups = computed<ConfigOption[][]>(() => {
  const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled;
  const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag;
  const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
+  const isObjectDetection =
+    useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection;

  return getTabGroups()
    .map((tabGroup) =>
      tabGroup.filter(
        (tabConfig) =>
          !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
-          !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
+          !((!allow3d || isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
          !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
-          !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco
+          !(!isAruco && tabConfig.tabName === "Aruco") &&
+          !(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out aruco unless we actually are doing Aruco
      )
    )
    .filter((it) => it.length); // Remove empty tab groups
--- a/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue
+++ b/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue
@@ -0,0 +1,35 @@
+<script setup lang="ts">
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { PipelineType } from "@/types/PipelineTypes";
+import PvSlider from "@/components/common/pv-slider.vue";
+import { computed, getCurrentInstance } from "vue";
+import { useStateStore } from "@/stores/StateStore";
+
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+
+const interactiveCols = computed(
+  () =>
+    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+)
+  ? 9
+  : 8;
+</script>
+
+<template>
+  <div v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+    <pv-slider
+      v-model="currentPipelineSettings.confidence"
+      class="pt-2"
+      :slider-cols="interactiveCols"
+      label="Confidence"
+      tooltip="The minimum confidence for a detection to be considered valid. Bigger numbers mean fewer but more probable detections are allowed through."
+      :min="0"
+      :max="1"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
+    />
+  </div>
+</template>
--- a/photon-client/src/components/dashboard/tabs/TargetsTab.vue
+++ b/photon-client/src/components/dashboard/tabs/TargetsTab.vue
@@ -48,6 +48,10 @@ const resetCurrentBuffer = () => {
              >
                Fiducial ID
              </th>
+              <template v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+                <th class="text-center white--text">Class</th>
+                <th class="text-center white--text">Confidence</th>
+              </template>
              <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                <th class="text-center white--text">Pitch &theta;&deg;</th>
                <th class="text-center white--text">Yaw &theta;&deg;</th>
@@ -85,6 +89,18 @@ const resetCurrentBuffer = () => {
              >
                {{ target.fiducialId }}
              </td>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ useStateStore().currentPipelineResults?.classNames[target.classId] }}
+              </td>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ target.confidence.toFixed(2) }}
+              </td>
              <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                <td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
                <td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>
--- a/photon-client/src/stores/settings/GeneralSettingsStore.ts
+++ b/photon-client/src/stores/settings/GeneralSettingsStore.ts
@@ -27,7 +27,8 @@ export const useSettingsStore = defineStore("settings", {
      gpuAcceleration: undefined,
      hardwareModel: undefined,
      hardwarePlatform: undefined,
-      mrCalWorking: true
+      mrCalWorking: true,
+      rknnSupported: false
    },
    network: {
      ntServerAddress: "",
@@ -99,7 +100,8 @@ export const useSettingsStore = defineStore("settings", {
        hardwareModel: data.general.hardwareModel || undefined,
        hardwarePlatform: data.general.hardwarePlatform || undefined,
        gpuAcceleration: data.general.gpuAcceleration || undefined,
-        mrCalWorking: data.general.mrCalWorking
+        mrCalWorking: data.general.mrCalWorking,
+        rknnSupported: data.general.rknnSupported
      };
      this.lighting = data.lighting;
      this.network = data.networkSettings;
--- a/photon-client/src/types/PhotonTrackingTypes.ts
+++ b/photon-client/src/types/PhotonTrackingTypes.ts
@@ -54,6 +54,8 @@ export interface PhotonTarget {
  ambiguity: number;
  // -1 if not set
  fiducialId: number;
+  confidence: number;
+  classId: number;
  // undefined if 3d isn't enabled
  pose?: Transform3d;
 }
@@ -70,4 +72,6 @@ export interface PipelineResult {
  targets: PhotonTarget[];
  // undefined if multitag failed or non-tag pipeline
  multitagResult?: MultitagResult;
+  // Object detection class names -- empty if not doing object detection
+  classNames: string[];
 }
--- a/photon-client/src/types/PipelineTypes.ts
+++ b/photon-client/src/types/PipelineTypes.ts
@@ -5,7 +5,8 @@ export enum PipelineType {
  Reflective = 2,
  ColoredShape = 3,
  AprilTag = 4,
-  Aruco = 5
+  Aruco = 5,
+  ObjectDetection = 6
 }

 export enum AprilTagFamily {
@@ -281,14 +282,39 @@ export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
  doSingleTargetAlways: false
 };

+export interface ObjectDetectionPipelineSettings extends PipelineSettings {
+  pipelineType: PipelineType.ObjectDetection;
+  confidence: number;
+  nms: number;
+  box_thresh: number;
+}
+export type ConfigurableObjectDetectionPipelineSettings = Partial<
+  Omit<ObjectDetectionPipelineSettings, "pipelineType">
+> &
+  ConfigurablePipelineSettings;
+export const DefaultObjectDetectionPipelineSettings: ObjectDetectionPipelineSettings = {
+  ...DefaultPipelineSettings,
+  pipelineType: PipelineType.ObjectDetection,
+  cameraGain: 20,
+  targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
+  ledMode: true,
+  outputShowMultipleTargets: false,
+  cameraExposure: 6,
+  confidence: 0.9,
+  nms: 0.45,
+  box_thresh: 0.25
+};
+
 export type ActivePipelineSettings =
  | ReflectivePipelineSettings
  | ColoredShapePipelineSettings
  | AprilTagPipelineSettings
-  | ArucoPipelineSettings;
+  | ArucoPipelineSettings
+  | ObjectDetectionPipelineSettings;

 export type ActiveConfigurablePipelineSettings =
  | ConfigurableReflectivePipelineSettings
  | ConfigurableColoredShapePipelineSettings
  | ConfigurableAprilTagPipelineSettings
-  | ConfigurableArucoPipelineSettings;
+  | ConfigurableArucoPipelineSettings
+  | ConfigurableObjectDetectionPipelineSettings;
--- a/photon-client/src/types/SettingTypes.ts
+++ b/photon-client/src/types/SettingTypes.ts
@@ -7,6 +7,7 @@ export interface GeneralSettings {
  hardwareModel?: string;
  hardwarePlatform?: string;
  mrCalWorking: boolean;
+  rknnSupported: boolean;
 }

 export interface MetricData {
--- a/photon-client/src/types/WebsocketDataTypes.ts
+++ b/photon-client/src/types/WebsocketDataTypes.ts
@@ -101,5 +101,6 @@ export enum WebsocketPipelineType {
  Reflective = 0,
  ColoredShape = 1,
  AprilTag = 2,
-  Aruco = 3
+  Aruco = 3,
+  ObjectDetection = 4
 }