Run multitag on coprocessor (#816)

photon-client/src/types/PhotonTrackingTypes.ts

@@ -1,12 +1,39 @@
-export interface Pose {
+export interface Transform3d {
   x: number;
   y: number;
   z: number;
-  angle_z: number;
   qw: number;
   qx: number;
   qy: number;
   qz: number;
+  angle_z: number;
+}
+
+export interface Quaternion {
+  X: number;
+  Y: number;
+  Z: number;
+  W: number;
+}
+
+export interface AprilTagFieldLayout {
+  field: {
+    length: number;
+    width: number;
+  };
+  tags: {
+    ID: number;
+    pose: {
+      translation: {
+        x: number;
+        y: number;
+        z: number;
+      };
+      rotation: {
+        quaternion: Quaternion;
+      };
+    };
+  }[];
 }
 
 export interface PhotonTarget {
@@ -19,11 +46,19 @@ export interface PhotonTarget {
   // -1 if not set
   fiducialId: number;
   // undefined if 3d isn't enabled
-  pose?: Pose;
+  pose?: Transform3d;
+}
+
+export interface MultitagResult {
+  bestTransform: Transform3d;
+  bestReprojectionError: number;
+  fiducialIDsUsed: number[];
 }
 
 export interface PipelineResult {
   fps: number;
   latency: number;
   targets: PhotonTarget[];
+  // undefined if multitag failed or non-tag pipeline
+  multitagResult?: MultitagResult;
 }

photon-client/src/types/PipelineTypes.ts

@@ -214,6 +214,8 @@ export interface AprilTagPipelineSettings extends PipelineSettings {
   debug: boolean;
   threads: number;
   tagFamily: AprilTagFamily;
+  doMultiTarget: boolean;
+  doSingleTargetAlways: boolean;
 }
 export type ConfigurableAprilTagPipelineSettings = Partial<
   Omit<AprilTagPipelineSettings, "pipelineType" | "hammingDist" | "debug">
@@ -236,7 +238,9 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = {
   refineEdges: true,
   debug: false,
   threads: 4,
-  tagFamily: AprilTagFamily.Family16h5
+  tagFamily: AprilTagFamily.Family16h5,
+  doMultiTarget: false,
+  doSingleTargetAlways: false
 };
 
 export interface ArucoPipelineSettings extends PipelineSettings {
@@ -270,6 +274,7 @@ export type ActivePipelineSettings =
   | ColoredShapePipelineSettings
   | AprilTagPipelineSettings
   | ArucoPipelineSettings;
+
 export type ActiveConfigurablePipelineSettings =
   | ConfigurableReflectivePipelineSettings
   | ConfigurableColoredShapePipelineSettings

photon-client/src/types/WebsocketDataTypes.ts

@@ -1,5 +1,6 @@
 import type { GeneralSettings, LightingSettings, LogLevel, MetricData, NetworkSettings } from "@/types/SettingTypes";
 import type { ActivePipelineSettings } from "@/types/PipelineTypes";
+import type { AprilTagFieldLayout, PipelineResult } from "@/types/PhotonTrackingTypes";
 
 export interface WebsocketLogMessage {
   logMessage: {
@@ -11,6 +12,7 @@ export interface WebsocketSettingsUpdate {
   general: Required<GeneralSettings>;
   lighting: Required<LightingSettings>;
   networkSettings: NetworkSettings;
+  atfl: AprilTagFieldLayout;
 }
 
 export interface WebsocketNumberPair {
@@ -60,31 +62,9 @@ export interface WebsocketNTUpdate {
   address?: string;
   clients?: number;
 }
-export type WebsocketPipelineResultUpdate = Record<
-  number,
-  {
-    fps: number;
-    latency: number;
-    targets: {
-      yaw: number;
-      pitch: number;
-      skew: number;
-      area: number;
-      ambiguity: number;
-      fiducialId: number;
-      pose: {
-        angle_z: number;
-        qw: number;
-        qx: number;
-        x: number;
-        qy: number;
-        y: number;
-        qz: number;
-        z: number;
-      };
-    }[];
-  }
->;
+
+// key is the index of the camera, value is that camera's result
+export type WebsocketPipelineResultUpdate = Record<string, PipelineResult>;
 
 export interface WebsocketCalibrationData {
   patternWidth: number;
@@ -106,7 +86,7 @@ export interface IncomingWebsocketData {
   updatePipelineResult?: WebsocketPipelineResultUpdate;
   networkInfo?: {
     possibleRios: string[];
-    deviceips: string[];
+    deviceIps: string[];
   };
   mutatePipelineSettings?: Partial<ActivePipelineSettings>;
   cameraIndex?: number; // Sent when mutating pipeline settings to check against currently active