Show board outliers in calibration info card (#1267)

build.gradle

@@ -43,7 +43,6 @@ ext {
     frcYear = "2026beta"
     mrcalVersion = "dev-v2025.0.0-3-g2dc275f";
 
-
     pubVersion = versionString
     isDev = pubVersion.startsWith("dev")
 

photon-client/src/components/app/photon-3d-visualizer.vue

@@ -8,8 +8,10 @@ import { onBeforeUnmount, onMounted, watchEffect } from "vue";
 const {
   ArrowHelper,
   BoxGeometry,
+  CameraHelper,
   Color,
   ConeGeometry,
+  Group,
   Mesh,
   MeshNormalMaterial,
   PerspectiveCamera,
@@ -20,6 +22,18 @@ const {
 } = await import("three");
 const { TrackballControls } = await import("three/examples/jsm/controls/TrackballControls");
 
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { createPerspectiveCamera } from "@/lib/ThreeUtils";
+import { useTheme } from "vuetify";
+
+const theme = useTheme();
+
+const calibrationCoeffs = useCameraSettingsStore().getCalibrationCoeffs(
+  useCameraSettingsStore().currentCameraSettings.validVideoFormats[
+    useCameraSettingsStore().currentPipelineSettings.cameraVideoModeIndex
+  ].resolution
+);
+
 const props = defineProps<{
   targets: PhotonTarget[];
 }>();
@@ -32,15 +46,18 @@ let controls: TrackballControls | undefined;
 let previousTargets: Object3D[] = [];
 const drawTargets = (targets: PhotonTarget[]) => {
   // Check here, since if we check in watchEffect this never gets called
-  if (scene === undefined || camera === undefined || renderer === undefined || controls === undefined) {
+  if (!scene || !camera || !renderer || !controls) {
     return;
   }
 
+  if (theme.global.name.value === "LightTheme") scene.background = new Color(0xa9a9a9);
+  else scene.background = new Color(0x000000);
+
   scene.remove(...previousTargets);
   previousTargets = [];
 
   targets.forEach((target) => {
-    if (target.pose === undefined) return;
+    if (!target.pose) return;
 
     const geometry = new BoxGeometry(0.3 / 5, 0.2, 0.2);
     const material = new MeshNormalMaterial();
@@ -70,6 +87,18 @@ const drawTargets = (targets: PhotonTarget[]) => {
     previousTargets.push(arrow);
   });
 
+  if (calibrationCoeffs) {
+    // And show camera frustum
+    const calibCamera = createPerspectiveCamera(calibrationCoeffs.resolution, calibrationCoeffs.cameraIntrinsics, 10);
+    const helper = new CameraHelper(calibCamera);
+    const helperGroup = new Group();
+    helperGroup.add(helper);
+    // Flip to +Z forward
+    helperGroup.rotateX(-Math.PI / 2.0);
+    helperGroup.rotateY(-Math.PI / 2.0);
+    previousTargets.push(helperGroup);
+  }
+
   if (previousTargets.length > 0) {
     scene.add(...previousTargets);
   }
@@ -78,7 +107,7 @@ const onWindowResize = () => {
   const container = document.getElementById("container");
   const canvas = document.getElementById("view");
 
-  if (container === null || canvas === null || camera === undefined || renderer === undefined) {
+  if (!container || !canvas || !camera || !renderer) {
     return;
   }
 
@@ -89,7 +118,7 @@ const onWindowResize = () => {
   renderer.setSize(canvas.clientWidth, canvas.clientHeight);
 };
 const resetCamFirstPerson = () => {
-  if (scene === undefined || camera === undefined || controls === undefined) {
+  if (!scene || !camera || !controls) {
     return;
   }
 
@@ -103,7 +132,7 @@ const resetCamFirstPerson = () => {
   }
 };
 const resetCamThirdPerson = () => {
-  if (scene === undefined || camera === undefined || controls === undefined) {
+  if (!scene || !camera || !controls) {
     return;
   }
 
@@ -122,10 +151,11 @@ onMounted(async () => {
   camera = new PerspectiveCamera(75, 800 / 800, 0.1, 1000);
 
   const canvas = document.getElementById("view");
-  if (canvas === null) return;
+  if (!canvas) return;
   renderer = new WebGLRenderer({ canvas: canvas });
 
-  scene.background = new Color(0xa9a9a9);
+  if (theme.global.name.value === "LightTheme") scene.background = new Color(0xa9a9a9);
+  else scene.background = new Color(0x000000);
 
   onWindowResize();
   window.addEventListener("resize", onWindowResize);
@@ -169,7 +199,7 @@ onMounted(async () => {
   controls.update();
 
   const animate = () => {
-    if (scene === undefined || camera === undefined || renderer === undefined || controls === undefined) {
+    if (!scene || !camera || !renderer || !controls) {
       return;
     }
 
@@ -192,18 +222,31 @@ watchEffect(() => {
 
 <template>
   <div id="container" style="width: 100%">
-    <v-row>
-      <v-col align-self="stretch" style="display: flex; justify-content: center">
-        <canvas id="view" />
+    <div class="d-flex flex-wrap pt-0 pb-2">
+      <v-col cols="12" md="6" class="pl-0">
+        <v-card-title class="pa-0"> Target Visualization </v-card-title>
       </v-col>
-    </v-row>
-    <v-row style="margin-bottom: 24px">
-      <v-col style="display: flex; justify-content: center">
-        <v-btn color="secondary" @click="resetCamFirstPerson"> First Person </v-btn>
+      <v-col cols="6" md="3" class="d-flex align-center pt-0 pt-md-3 pl-6 pl-md-3">
+        <v-btn
+          style="width: 100%"
+          color="buttonActive"
+          :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
+          @click="resetCamFirstPerson"
+        >
+          First Person
+        </v-btn>
       </v-col>
-      <v-col style="display: flex; justify-content: center">
-        <v-btn color="secondary" @click="resetCamThirdPerson"> Third Person </v-btn>
+      <v-col cols="6" md="3" class="d-flex align-center pt-0 pt-md-3 pr-0">
+        <v-btn
+          style="width: 100%"
+          color="buttonActive"
+          :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
+          @click="resetCamThirdPerson"
+        >
+          Third Person
+        </v-btn>
       </v-col>
-    </v-row>
+    </div>
+    <canvas id="view" class="w-100" />
   </div>
 </template>

photon-client/src/components/app/photon-calibration-visualizer.vue

@@ -0,0 +1,371 @@
+<script setup lang="ts">
+import { onBeforeUnmount, onMounted, ref, watch, watchEffect, type Ref } from "vue";
+const {
+  AmbientLight,
+  AxesHelper,
+  BoxGeometry,
+  CameraHelper,
+  Color,
+  ConeGeometry,
+  Group,
+  Mesh,
+  MeshNormalMaterial,
+  MeshPhongMaterial,
+  PerspectiveCamera,
+  Scene,
+  SphereGeometry,
+  WebGLRenderer
+} = await import("three");
+const { TrackballControls } = await import("three/examples/jsm/controls/TrackballControls");
+import type { BoardObservation, CameraCalibrationResult } from "@/types/SettingTypes";
+import axios from "axios";
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { useTheme } from "vuetify";
+import { createPerspectiveCamera } from "@/lib/ThreeUtils";
+
+const theme = useTheme();
+
+const props = defineProps<{
+  cameraUniqueName: string;
+  resolution: { width: number; height: number };
+  title: string;
+}>();
+
+let scene: Scene | undefined;
+let camera: PerspectiveCamera | undefined;
+let renderer: WebGLRenderer | undefined;
+let controls: TrackballControls | undefined;
+
+const createChessboard = (obs: BoardObservation, cal: CameraCalibrationResult): Group => {
+  const group = new Group();
+
+  if (obs.locationInImageSpace.length === 0) return group;
+
+  // Add corner spheres
+  obs.locationInObjectSpace.forEach((corner, idx) => {
+    if (corner.x < 0 || corner.y < 0) return;
+
+    const isOutlier = !obs.cornersUsed[idx];
+
+    const color = isOutlier ? 0xff3333 : 0x33ff33;
+
+    const sphereGeom = new SphereGeometry(cal.calobjectSpacing / 8, 8, 8);
+    const sphereMat = new MeshPhongMaterial({
+      color: color,
+      opacity: 1,
+      transparent: !isOutlier
+    });
+    const sphere = new Mesh(sphereGeom, sphereMat);
+    sphere.position.set(corner.x, corner.y, corner.z);
+    group.add(sphere);
+  });
+
+  return group;
+};
+
+let previousTargets: Object3D[] = [];
+let baseAspect: number | undefined;
+const drawCalibration = (cal: CameraCalibrationResult | null) => {
+  // Check here, since if we check in watchEffect this never gets called
+  if (!cal || !scene || !camera || !renderer || !controls) {
+    return;
+  }
+
+  scene.remove(...previousTargets);
+  previousTargets = [];
+
+  // Draw all chessboards with transparency
+  cal.observations.forEach((obs) => {
+    const pose = obs.optimisedCameraToObject;
+
+    // Create chessboard
+    const board = createChessboard(obs, cal);
+    board.userData.isCalibrationObject = true;
+
+    // Apply transform from camera to chessboard
+    const pos = pose.translation;
+    board.position.set(pos.x, pos.y, pos.z);
+
+    if (pose.rotation.quaternion) {
+      const q = pose.rotation.quaternion;
+      board.quaternion.set(q.X, q.Y, q.Z, q.W);
+    }
+
+    previousTargets.push(board);
+  });
+
+  // And show camera frustum
+  const calibCamera = createPerspectiveCamera(props.resolution, cal.cameraIntrinsics);
+  const helper = new CameraHelper(calibCamera);
+
+  // Flip to +Z forward
+  const helperGroup = new Group();
+  helperGroup.add(helper);
+  helperGroup.rotateY(Math.PI);
+
+  previousTargets.push(helperGroup);
+
+  if (previousTargets.length > 0) {
+    scene.add(...previousTargets);
+  }
+};
+
+const calibrationData: Ref<CameraCalibrationResult | null> = ref(null);
+const isLoading: Ref<boolean> = ref(true);
+const error: Ref<string | null> = ref(null);
+
+const fetchCalibrationData = async () => {
+  isLoading.value = true;
+  error.value = null;
+
+  try {
+    const response = await axios.get("/settings/camera/getCalibration", {
+      params: {
+        cameraUniqueName: props.cameraUniqueName,
+        width: props.resolution.width,
+        height: props.resolution.height
+      }
+    });
+    calibrationData.value = response.data;
+  } catch (err) {
+    console.error("Failed to fetch calibration data:", err);
+    error.value = "Failed to load calibration data";
+  } finally {
+    isLoading.value = false;
+  }
+};
+
+const onWindowResize = () => {
+  const container = document.getElementById("container");
+  const canvas = document.getElementById("view");
+
+  if (!container || !canvas || !camera || !renderer) {
+    return;
+  }
+
+  // Compute a concrete width from the container and derive height from a
+  // stable base aspect ratio (calculated on mount) to avoid feedback loops
+  // where updating canvas size changes container size while resizing
+  const width = Math.max(1, Math.floor(container.clientWidth));
+  let height: number;
+  if (baseAspect && baseAspect > 0) {
+    height = Math.max(1, Math.floor(width / baseAspect));
+  } else {
+    height = Math.max(1, Math.floor(container.clientHeight));
+  }
+
+  // Use updateStyle=false so Three.js does not write to canvas style,
+  // which can affect layout and re-trigger resize events
+  renderer.setSize(width, height, false);
+  camera.aspect = width / height;
+  camera.updateProjectionMatrix();
+};
+
+const resetCamFirstPerson = () => {
+  if (!scene || !camera || !controls) {
+    return;
+  }
+
+  controls.reset();
+  camera.position.set(0, 0, 0.05);
+  camera.up.set(0, -1, 0);
+  controls.target.set(0.0, 0.0, 1.0);
+  controls.update();
+  if (previousTargets.length > 0) {
+    scene.add(...previousTargets);
+  }
+};
+
+const resetCamThirdPerson = () => {
+  if (!scene || !camera || !controls) {
+    return;
+  }
+
+  controls.reset();
+  camera.position.set(-0.3, -0.2, -0.3);
+  camera.up.set(0, -1, 0);
+  controls.target.set(0.0, 0.0, 0.4);
+  controls.update();
+  if (previousTargets.length > 0) {
+    scene.add(...previousTargets);
+  }
+};
+
+let animationFrameId: number | null = null;
+
+onMounted(async () => {
+  // Grab data first off
+  fetchCalibrationData();
+
+  scene = new Scene();
+  camera = new PerspectiveCamera(75, 800 / 800, 0.1, 1000);
+
+  const canvas = document.getElementById("view");
+  if (!canvas) return;
+  renderer = new WebGLRenderer({ canvas: canvas });
+
+  // Add lights
+  const ambientLight = new AmbientLight(0xffffff, 0.6);
+  scene.add(ambientLight);
+
+  if (theme.global.name.value === "LightTheme") scene.background = new Color(0xa9a9a9);
+  else scene.background = new Color(0x000000);
+
+  // Initialize a stable aspect ratio so subsequent resize events derive
+  // height from width, avoiding layout feedback during continuous resizing
+  try {
+    const initWidth = Math.max(1, Math.floor(document.getElementById("container")?.clientWidth || 1));
+    const initHeight = Math.max(1, Math.floor(document.getElementById("container")?.clientHeight || 1));
+    baseAspect = initWidth / Math.max(1, initHeight);
+  } catch {
+    baseAspect = undefined;
+  }
+
+  onWindowResize();
+  window.addEventListener("resize", onWindowResize);
+
+  const referenceFrameCues: Object3D[] = [];
+
+  // Draw the reference frame
+  referenceFrameCues.push(new AxesHelper(0.3));
+
+  // Draw the Camera Body
+  const camSize = 0.04;
+  const camBodyGeometry = new BoxGeometry(camSize, camSize, camSize);
+  const camLensGeometry = new ConeGeometry(camSize * 0.4, camSize * 0.8, 30);
+  const camMaterial = new MeshNormalMaterial();
+  const camBody = new Mesh(camBodyGeometry, camMaterial);
+  const camLens = new Mesh(camLensGeometry, camMaterial);
+  camBody.position.set(0, 0, 0);
+  camLens.rotateX(-Math.PI / 2);
+  camLens.position.set(0, 0, camSize * 0.8);
+  referenceFrameCues.push(camBody);
+  referenceFrameCues.push(camLens);
+
+  controls = new TrackballControls(camera, renderer.domElement);
+  controls.rotateSpeed = 1.0;
+  controls.zoomSpeed = 1.2;
+  controls.panSpeed = 0.8;
+  controls.noZoom = false;
+  controls.noPan = false;
+  controls.staticMoving = true;
+  controls.dynamicDampingFactor = 0.3;
+
+  scene.add(...referenceFrameCues);
+  resetCamThirdPerson();
+
+  controls.update();
+
+  const animate = () => {
+    if (!scene || !camera || !renderer || !controls) {
+      return;
+    }
+
+    animationFrameId = requestAnimationFrame(animate);
+
+    controls.update();
+    renderer.render(scene, camera);
+  };
+
+  animate();
+});
+
+const cleanup = () => {
+  window.removeEventListener("resize", onWindowResize);
+
+  if (animationFrameId) {
+    cancelAnimationFrame(animationFrameId);
+  }
+
+  if (controls) {
+    controls.dispose();
+  }
+
+  if (renderer) {
+    renderer.dispose();
+    renderer.forceContextLoss();
+  }
+
+  if (scene) {
+    scene.traverse((object) => {
+      if (object instanceof Mesh) {
+        object.geometry?.dispose();
+        if (object.material) {
+          if (Array.isArray(object.material)) {
+            object.material.forEach((material) => material.dispose());
+          } else {
+            object.material.dispose();
+          }
+        }
+      }
+    });
+  }
+
+  scene = undefined;
+  camera = undefined;
+  renderer = undefined;
+  controls = undefined;
+  previousTargets = [];
+};
+
+onBeforeUnmount(cleanup);
+
+// If hot-reloading, cleanup on hot reload
+if (import.meta.hot) {
+  import.meta.hot.dispose(() => {
+    cleanup();
+  });
+}
+
+watchEffect(() => {
+  drawCalibration(calibrationData.value);
+});
+
+watch(
+  () => [
+    props.cameraUniqueName,
+    props.resolution.width,
+    props.resolution.height,
+    useCameraSettingsStore().getCalibrationCoeffs(props.resolution)
+  ],
+  () => {
+    console.log("Camera or resolution changed, refetching calibration");
+    fetchCalibrationData();
+  }
+);
+</script>
+
+<template>
+  <div style="width: 100%; height: 100%" class="d-flex flex-column">
+    <div class="d-flex flex-wrap pt-0 pb-2">
+      <v-col cols="12" md="6" class="pl-0">
+        <v-card-title class="pa-0">
+          {{ props.title }}
+        </v-card-title>
+      </v-col>
+      <v-col cols="6" md="3" class="d-flex align-center pt-0 pt-md-3 pl-6 pl-md-3">
+        <v-btn
+          style="width: 100%"
+          color="buttonActive"
+          :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
+          @click="resetCamFirstPerson"
+        >
+          First Person
+        </v-btn>
+      </v-col>
+      <v-col cols="6" md="3" class="d-flex align-center pt-0 pt-md-3 pr-0">
+        <v-btn
+          style="width: 100%"
+          color="buttonActive"
+          :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
+          @click="resetCamThirdPerson"
+        >
+          Third Person
+        </v-btn>
+      </v-col>
+    </div>
+    <div id="container" style="flex: 1 1 auto">
+      <canvas id="view" class="w-100 h-100" />
+    </div>
+  </div>
+</template>

photon-client/src/components/cameras/CameraCalibrationCard.vue

@@ -38,7 +38,8 @@ const getUniqueVideoFormatsByResolution = (): VideoFormat[] => {
     if (!skip) {
       const calib = useCameraSettingsStore().getCalibrationCoeffs(format.resolution);
       if (calib !== undefined) {
-        // For each error, square it, sum the squares, and divide by total points N
+        // Mean overall reprojection error
+        // Calculated as average of each observation's mean error
         if (calib.meanErrors.length)
           format.mean = calib.meanErrors.reduce((a, b) => a + b, 0) / calib.meanErrors.length;
         else format.mean = NaN;
@@ -249,27 +250,31 @@ const setSelectedVideoFormat = (format: VideoFormat) => {
               <th>Horizontal FOV</th>
               <th>Vertical FOV</th>
               <th>Diagonal FOV</th>
-              <th>Info</th>
             </tr>
           </thead>
           <tbody style="cursor: pointer">
-            <tr v-for="(value, index) in getUniqueVideoFormatsByResolution()" :key="index">
-              <td>{{ getResolutionString(value.resolution) }}</td>
-              <td>
-                {{ value.mean !== undefined ? (isNaN(value.mean) ? "Unknown" : value.mean.toFixed(2) + "px") : "-" }}
-              </td>
-              <td>{{ value.horizontalFOV !== undefined ? value.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
-              <td>{{ value.verticalFOV !== undefined ? value.verticalFOV.toFixed(2) + "°" : "-" }}</td>
-              <td>{{ value.diagonalFOV !== undefined ? value.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
-              <v-tooltip location="bottom">
-                <template #activator="{ props }">
-                  <td v-bind="props" @click="setSelectedVideoFormat(value)">
-                    <v-icon size="small" color="primary">mdi-information</v-icon>
+            <v-tooltip
+              v-for="(value, index) in getUniqueVideoFormatsByResolution()"
+              :key="index"
+              transition=""
+              location="bottom"
+              :open-delay="100"
+            >
+              <template #activator="{ props }">
+                <tr :key="index" v-bind="props" @click="setSelectedVideoFormat(value)">
+                  <td>{{ getResolutionString(value.resolution) }}</td>
+                  <td>
+                    {{
+                      value.mean !== undefined ? (isNaN(value.mean) ? "Unknown" : value.mean.toFixed(2) + "px") : "-"
+                    }}
                   </td>
-                </template>
-                <span>View calibration information</span>
-              </v-tooltip>
-            </tr>
+                  <td>{{ value.horizontalFOV !== undefined ? value.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
+                  <td>{{ value.verticalFOV !== undefined ? value.verticalFOV.toFixed(2) + "°" : "-" }}</td>
+                  <td>{{ value.diagonalFOV !== undefined ? value.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
+                </tr>
+              </template>
+              <span>View calibration information</span>
+            </v-tooltip>
           </tbody>
         </v-table>
       </v-card-text>

photon-client/src/components/cameras/CameraCalibrationInfoCard.vue

@@ -1,4 +1,5 @@
 <script setup lang="ts">
+import PhotonCalibrationVisualizer from "@/components/app/photon-calibration-visualizer.vue";
 import type { CameraCalibrationResult, VideoFormat } from "@/types/SettingTypes";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { useStateStore } from "@/stores/StateStore";
@@ -8,7 +9,6 @@ import { useTheme } from "vuetify";
 import PvDeleteModal from "@/components/common/pv-delete-modal.vue";
 
 const theme = useTheme();
-
 const props = defineProps<{
   videoFormat: VideoFormat;
 }>();
@@ -79,8 +79,10 @@ const importCalibration = async () => {
 };
 
 interface ObservationDetails {
-  mean: number;
   index: number;
+  mean: number;
+  numOutliers: number;
+  numMissing: number;
 }
 
 const currentCalibrationCoeffs = computed<CameraCalibrationResult | undefined>(() =>
@@ -92,7 +94,9 @@ const getObservationDetails = (): ObservationDetails[] | undefined => {
 
   return coefficients?.meanErrors.map((m, i) => ({
     index: i,
-    mean: parseFloat(m.toFixed(2))
+    mean: parseFloat(m.toFixed(2)),
+    numOutliers: coefficients.numOutliers[i],
+    numMissing: coefficients.numMissing[i]
   }));
 };
 
@@ -101,213 +105,236 @@ const exportCalibrationURL = computed<string>(() =>
 );
 const calibrationImageURL = (index: number) =>
   useCameraSettingsStore().getCalImageUrl(inject<string>("backendHost") as string, props.videoFormat.resolution, index);
+
+const tab = ref("details");
+const viewingImg = ref(0);
 </script>
+
 <template>
   <v-card color="surface" dark>
-    <div class="d-flex flex-wrap pt-2 pl-2 pr-2 align-center">
-      <v-col cols="12" md="6">
-        <v-card-title class="pa-0"> Calibration Details </v-card-title>
-      </v-col>
-      <v-col cols="12" md="6" class="d-flex align-center pt-0 pt-md-3">
-        <v-btn
-          color="buttonPassive"
-          class="mr-2"
-          style="flex: 1"
-          :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
-          @click="openUploadPhotonCalibJsonPrompt"
-        >
-          <v-icon start size="large">mdi-import</v-icon>
-          <span>Import</span>
-        </v-btn>
-        <input
-          ref="importCalibrationFromPhotonJson"
-          type="file"
-          accept=".json"
-          style="display: none"
-          @change="importCalibration"
-        />
-        <v-btn
-          color="buttonPassive"
-          class="mr-2"
-          :disabled="!currentCalibrationCoeffs"
-          style="flex: 1"
-          :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
-          @click="openExportCalibrationPrompt"
-        >
-          <v-icon start size="large">mdi-export</v-icon>
-          <span>Export</span>
-        </v-btn>
-        <a
-          ref="exportCalibration"
-          style="color: black; text-decoration: none; display: none"
-          :href="exportCalibrationURL"
-          target="_blank"
-        />
-        <v-btn
-          color="error"
-          :disabled="!currentCalibrationCoeffs"
-          style="flex: 1"
-          :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
-          @click="() => (confirmRemoveDialog = { show: true, vf: props.videoFormat })"
-        >
-          <v-icon start size="large">mdi-delete</v-icon>
-          <span>Delete</span>
-        </v-btn>
-      </v-col>
-    </div>
-    <v-card-title class="pt-0 pb-0"
-      >{{ useCameraSettingsStore().currentCameraName }}@{{ getResolutionString(videoFormat.resolution) }}</v-card-title
-    >
-    <v-card-text v-if="!currentCalibrationCoeffs">
-      <v-alert
-        class="pt-3 pb-3"
-        color="primary"
-        density="compact"
-        text="The selected video format has not been calibrated."
-        icon="mdi-alert-circle-outline"
-        :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'tonal'"
-      />
-    </v-card-text>
-    <v-card-text class="pt-0">
-      <v-table density="compact" style="width: 100%">
-        <template #default>
-          <thead>
-            <tr>
-              <th class="text-left">Name</th>
-              <th class="text-left">Value</th>
-            </tr>
-          </thead>
-          <tbody>
-            <tr>
-              <td>Fx</td>
-              <td>
-                {{
-                  useCameraSettingsStore()
-                    .getCalibrationCoeffs(props.videoFormat.resolution)
-                    ?.cameraIntrinsics.data[0].toFixed(2) || 0.0
-                }}
-                mm
-              </td>
-            </tr>
-            <tr>
-              <td>Fy</td>
-              <td>
-                {{
-                  useCameraSettingsStore()
-                    .getCalibrationCoeffs(props.videoFormat.resolution)
-                    ?.cameraIntrinsics.data[4].toFixed(2) || 0.0
-                }}
-                mm
-              </td>
-            </tr>
-            <tr>
-              <td>Cx</td>
-              <td>
-                {{
-                  useCameraSettingsStore()
-                    .getCalibrationCoeffs(props.videoFormat.resolution)
-                    ?.cameraIntrinsics.data[2].toFixed(2) || 0.0
-                }}
-                px
-              </td>
-            </tr>
-            <tr>
-              <td>Cy</td>
-              <td>
-                {{
-                  useCameraSettingsStore()
-                    .getCalibrationCoeffs(props.videoFormat.resolution)
-                    ?.cameraIntrinsics.data[5].toFixed(2) || 0.0
-                }}
-                px
-              </td>
-            </tr>
-            <tr>
-              <td>Distortion</td>
-              <td>
-                {{
-                  useCameraSettingsStore()
-                    .getCalibrationCoeffs(props.videoFormat.resolution)
-                    ?.distCoeffs.data.map((it) => parseFloat(it.toFixed(3))) || []
-                }}
-              </td>
-            </tr>
-            <tr>
-              <td>Mean Err</td>
-              <td>
-                {{
-                  videoFormat.mean !== undefined
-                    ? isNaN(videoFormat.mean)
-                      ? "NaN"
-                      : videoFormat.mean.toFixed(2) + "px"
-                    : "-"
-                }}
-              </td>
-            </tr>
-            <tr>
-              <td>Horizontal FOV</td>
-              <td>
-                {{ videoFormat.horizontalFOV !== undefined ? videoFormat.horizontalFOV.toFixed(2) + "°" : "-" }}
-              </td>
-            </tr>
-            <tr>
-              <td>Vertical FOV</td>
-              <td>{{ videoFormat.verticalFOV !== undefined ? videoFormat.verticalFOV.toFixed(2) + "°" : "-" }}</td>
-            </tr>
-            <tr>
-              <td>Diagonal FOV</td>
-              <td>{{ videoFormat.diagonalFOV !== undefined ? videoFormat.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
-            </tr>
-            <!-- Board warp, only shown for mrcal-calibrated cameras -->
-            <tr v-if="currentCalibrationCoeffs?.calobjectWarp?.length === 2">
-              <td>Board warp, X/Y</td>
-              <td>
-                {{
-                  useCameraSettingsStore()
-                    .getCalibrationCoeffs(props.videoFormat.resolution)
-                    ?.calobjectWarp?.map((it) => (it * 1000).toFixed(2) + " mm")
-                    .join(" / ")
-                }}
-              </td>
-            </tr>
-          </tbody>
-        </template>
-      </v-table>
-    </v-card-text>
-    <v-card-title v-if="currentCalibrationCoeffs" class="pt-0 pb-0">Individual Observations</v-card-title>
-    <v-card-text v-if="currentCalibrationCoeffs" class="pt-0">
-      <v-data-table
-        density="compact"
-        style="width: 100%"
-        :headers="[
-          { title: 'Observation Id', key: 'index' },
-          { title: 'Mean Reprojection Error', key: 'mean' },
-          { title: '', key: 'data-table-expand' }
-        ]"
-        :items="getObservationDetails()"
-        item-value="index"
-        show-expand
-      >
-        <template #item.data-table-expand="{ internalItem, toggleExpand }">
+    <v-card-title class="pb-2">
+      <div class="d-flex flex-wrap">
+        <v-col cols="12" md="6" class="pa-0">
+          <v-card-title class="pa-0"> Calibration Details </v-card-title>
+        </v-col>
+        <v-col cols="6" md="3" class="d-flex align-center pt-0 pb-0 pl-0">
           <v-btn
-            icon="mdi-eye"
-            class="text-none"
-            color="medium-emphasis"
-            size="small"
-            variant="text"
-            slim
-            @click="toggleExpand(internalItem)"
-          ></v-btn>
-        </template>
+            color="buttonPassive"
+            style="width: 100%"
+            :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
+            @click="openUploadPhotonCalibJsonPrompt"
+          >
+            <v-icon start size="large"> mdi-import</v-icon>
+            <span>Import</span>
+          </v-btn>
+          <input
+            ref="importCalibrationFromPhotonJson"
+            type="file"
+            accept=".json"
+            style="display: none"
+            @change="importCalibration"
+          />
+        </v-col>
+        <v-col cols="6" md="3" class="d-flex align-center pt-0 pb-0 pr-0">
+          <v-btn
+            color="buttonPassive"
+            :disabled="!currentCalibrationCoeffs"
+            style="width: 100%"
+            :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'outlined'"
+            @click="openExportCalibrationPrompt"
+          >
+            <v-icon start size="large">mdi-export</v-icon>
+            <span>Export</span>
+          </v-btn>
+          <a
+            ref="exportCalibration"
+            style="color: black; text-decoration: none; display: none"
+            :href="exportCalibrationURL"
+            target="_blank"
+          />
+        </v-col>
+      </div>
+    </v-card-title>
 
-        <template #expanded-row="{ columns, item }">
-          <td :colspan="columns.length">
-            <div style="display: flex; justify-content: center; width: 100%">
-              <img :src="calibrationImageURL(item.index)" alt="observation image" class="snapshot-preview pt-2 pb-2" />
-            </div>
-          </td>
-        </template>
-      </v-data-table>
+    <v-card-text class="d-flex flex-row pt-0">
+      <v-col cols="4" class="pa-0">
+        <v-tabs v-model="tab" grow bg-color="surface" height="48" slider-color="buttonActive">
+          <v-tab key="details" value="details">Details</v-tab>
+          <v-tab key="observations" value="observations">Observations</v-tab>
+        </v-tabs>
+        <v-tabs-window v-model="tab" class="pt-3">
+          <v-tabs-window-item key="details" value="details">
+            <v-table style="width: 100%" density="compact">
+              <template #default>
+                <tbody>
+                  <tr>
+                    <td>Camera</td>
+                    <td>
+                      {{ useCameraSettingsStore().currentCameraName }}
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Resolution</td>
+                    <td>
+                      {{ getResolutionString(videoFormat.resolution) }}
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Fx</td>
+                    <td>
+                      {{
+                        useCameraSettingsStore()
+                          .getCalibrationCoeffs(props.videoFormat.resolution)
+                          ?.cameraIntrinsics.data[0].toFixed(2) || 0.0
+                      }}
+                      mm
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Fy</td>
+                    <td>
+                      {{
+                        useCameraSettingsStore()
+                          .getCalibrationCoeffs(props.videoFormat.resolution)
+                          ?.cameraIntrinsics.data[4].toFixed(2) || 0.0
+                      }}
+                      mm
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Cx</td>
+                    <td>
+                      {{
+                        useCameraSettingsStore()
+                          .getCalibrationCoeffs(props.videoFormat.resolution)
+                          ?.cameraIntrinsics.data[2].toFixed(2) || 0.0
+                      }}
+                      px
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Cy</td>
+                    <td>
+                      {{
+                        useCameraSettingsStore()
+                          .getCalibrationCoeffs(props.videoFormat.resolution)
+                          ?.cameraIntrinsics.data[5].toFixed(2) || 0.0
+                      }}
+                      px
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Distortion</td>
+                    <td>
+                      {{
+                        useCameraSettingsStore()
+                          .getCalibrationCoeffs(props.videoFormat.resolution)
+                          ?.distCoeffs.data.map((it) => parseFloat(it.toFixed(3))) || []
+                      }}
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Mean Err</td>
+                    <td>
+                      {{
+                        videoFormat.mean !== undefined
+                          ? isNaN(videoFormat.mean)
+                            ? "NaN"
+                            : videoFormat.mean.toFixed(2) + "px"
+                          : "-"
+                      }}
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Horizontal FOV</td>
+                    <td>
+                      {{ videoFormat.horizontalFOV !== undefined ? videoFormat.horizontalFOV.toFixed(2) + "°" : "-" }}
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Vertical FOV</td>
+                    <td>
+                      {{ videoFormat.verticalFOV !== undefined ? videoFormat.verticalFOV.toFixed(2) + "°" : "-" }}
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>Diagonal FOV</td>
+                    <td>
+                      {{ videoFormat.diagonalFOV !== undefined ? videoFormat.diagonalFOV.toFixed(2) + "°" : "-" }}
+                    </td>
+                  </tr>
+                  <!-- Board warp, only shown for mrcal-calibrated cameras -->
+                  <tr v-if="currentCalibrationCoeffs?.calobjectWarp?.length === 2">
+                    <td>Board warp, X/Y</td>
+                    <td>
+                      {{
+                        currentCalibrationCoeffs?.calobjectWarp?.map((it) => (it * 1000).toFixed(2) + " mm").join(" / ")
+                      }}
+                    </td>
+                  </tr>
+                </tbody>
+              </template>
+            </v-table>
+          </v-tabs-window-item>
+          <v-tabs-window-item key="observations" value="observations">
+            <v-data-table
+              id="observations-table"
+              items-per-page-text="Page size:"
+              density="compact"
+              style="width: 100%"
+              :headers="[
+                { title: 'Id', key: 'index' },
+                { title: 'Mean Reprojection Error', key: 'mean' }
+              ]"
+              :items="getObservationDetails()"
+              item-value="index"
+              show-expand
+            >
+              <template #item.data-table-expand="{ internalItem }">
+                <v-btn
+                  class="text-none"
+                  size="small"
+                  variant="text"
+                  slim
+                  rounded
+                  @click="viewingImg = internalItem.index"
+                >
+                  <v-icon
+                    size="large"
+                    :color="viewingImg === internalItem.index ? 'buttonActive' : 'rgba(255, 255, 255, 0.7)'"
+                    >mdi-eye</v-icon
+                  >
+                </v-btn>
+              </template>
+            </v-data-table>
+          </v-tabs-window-item>
+        </v-tabs-window>
+      </v-col>
+      <v-col cols="8" class="pa-0 pl-6">
+        <v-card-text class="pa-0 fill-height d-flex justify-center align-center">
+          <div v-if="!currentCalibrationCoeffs">
+            <v-alert
+              class="pt-3 pb-3"
+              color="primary"
+              text="The selected video format has not been calibrated."
+              icon="mdi-alert-circle-outline"
+              :variant="theme.global.name.value === 'LightTheme' ? 'elevated' : 'tonal'"
+            />
+          </div>
+          <Suspense v-else-if="tab === 'details'">
+            <!-- Allows us to import three js when it's actually needed  -->
+            <PhotonCalibrationVisualizer
+              :camera-unique-name="useCameraSettingsStore().currentCameraSettings.uniqueName"
+              :resolution="props.videoFormat.resolution"
+              title="Camera to Board Transforms"
+            />
+            <template #fallback> Loading... </template>
+          </Suspense>
+          <div v-else style="display: flex; justify-content: center; width: 100%">
+            <img :src="calibrationImageURL(viewingImg)" alt="observation image" class="snapshot-preview pt-2 pb-2" />
+          </div>
+        </v-card-text>
+      </v-col>
     </v-card-text>
   </v-card>
 
@@ -322,11 +349,7 @@ const calibrationImageURL = (index: number) =>
 
 <style scoped>
 .snapshot-preview {
-  max-width: 55%;
-}
-@media only screen and (max-width: 512px) {
-  .snapshot-preview {
-    max-width: 100%;
-  }
+  max-width: 100%;
+  max-height: 100%;
 }
 </style>

photon-client/src/components/dashboard/tabs/Map3DTab.vue

@@ -9,20 +9,11 @@ const trackedTargets = computed<PhotonTarget[]>(() => useStateStore().currentPip
 
 <template>
   <div>
-    <v-row style="width: 100%">
-      <v-col>
-        <span class="text-white">Target Visualization</span>
-      </v-col>
-    </v-row>
-    <v-row style="width: 100%">
-      <v-col style="display: flex; align-items: center; justify-content: center">
-        <Suspense>
-          <!-- Allows us to import three js when it's actually needed  -->
-          <photon3d-visualizer :targets="trackedTargets" />
+    <Suspense>
+      <!-- Allows us to import three js when it's actually needed  -->
+      <photon3d-visualizer :targets="trackedTargets" />
 
-          <template #fallback> Loading... </template>
-        </Suspense>
-      </v-col>
-    </v-row>
+      <template #fallback> Loading... </template>
+    </Suspense>
   </div>
 </template>

photon-client/src/lib/ThreeUtils.ts

@@ -0,0 +1,23 @@
+import type { JsonMatOfDouble, Resolution } from "@/types/SettingTypes";
+const { PerspectiveCamera } = await import("three");
+
+/**
+ * Convert a camera intrinsics matrix and image resolution to a Three.js PerspectiveCamera. This assumes no skew and square pixels (same focal length in x and y), which is a sane assumption for most FRC cameras
+ *
+ * @param resolution video mode width/height
+ * @param intrinsicsCore camera intrinsics from the backend, row-major
+ * @returns a Three.js PerspectiveCamera matching the provided intrinsics
+ */
+export const createPerspectiveCamera = (
+  resolution: Resolution,
+  intrinsicsCore: JsonMatOfDouble,
+  frustumMax: number = 1
+) => {
+  const imageWidth = resolution.width;
+  const imageHeight = resolution.height;
+  const focalLengthY = intrinsicsCore.data[4];
+  const fovY = 2 * Math.atan(imageHeight / (2 * focalLengthY)) * (180 / Math.PI);
+  const aspect = imageWidth / imageHeight;
+
+  return new PerspectiveCamera(fovY, aspect, 0.1, frustumMax);
+};

photon-client/src/types/SettingTypes.ts

@@ -191,7 +191,7 @@ export interface BoardObservation {
   locationInImageSpace: CvPoint[];
   reprojectionErrors: CvPoint[];
   optimisedCameraToObject: Pose3d;
-  includeObservationInCalibration: boolean;
+  cornersUsed: boolean[];
   snapshotName: string;
   snapshotData: JsonImageMat;
 }
@@ -202,9 +202,15 @@ export interface CameraCalibrationResult {
   distCoeffs: JsonMatOfDouble;
   observations: BoardObservation[];
   calobjectWarp?: number[];
-  // We might have to omit observations for bandwidth, so backend will send us this
+  calobjectSize: { width: number; height: number };
+  calobjectSpacing: number;
+  lensModel: string;
+
+  // We have to omit observations for bandwidth, so backend will send us this from UICameraCalibrationCoefficients
   numSnapshots: number;
   meanErrors: number[];
+  numMissing: number[];
+  numOutliers: number[];
 }
 
 export enum ValidQuirks {
@@ -315,7 +321,7 @@ export const PlaceholderCameraSettings: UiCameraConfiguration = reactive({
         rows: 1,
         cols: 1,
         type: 1,
-        data: [1, 2, 3, 4, 5, 6, 7, 8, 9]
+        data: [600, 0, 1920 / 2, 0, 600, 1080 / 2, 0, 0, 1]
       },
       distCoeffs: {
         rows: 1,
@@ -325,28 +331,72 @@ export const PlaceholderCameraSettings: UiCameraConfiguration = reactive({
       },
       observations: [
         {
-          locationInImageSpace: [
-            { x: 100, y: 100 },
-            { x: 210, y: 100 },
-            { x: 320, y: 101 }
+          locationInObjectSpace: [
+            {
+              x: 0,
+              y: 0,
+              z: 0
+            },
+            {
+              x: 0.02539999969303608,
+              y: 0,
+              z: 0
+            },
+            {
+              x: 0.05079999938607216,
+              y: 0,
+              z: 0
+            }
+          ],
+          locationInImageSpace: [
+            {
+              x: 57.062007904052734,
+              y: 108.12601470947266
+            },
+            {
+              x: 108.72974395751953,
+              y: 108.0336685180664
+            },
+            {
+              x: 158.78118896484375,
+              y: 107.8104019165039
+            }
           ],
-          locationInObjectSpace: [{ x: 0, y: 0, z: 0 }],
           optimisedCameraToObject: {
-            translation: { x: 1, y: 2, z: 3 },
-            rotation: { quaternion: { W: 1, X: 0, Y: 0, Z: 0 } }
+            translation: {
+              x: -0.28942385915178886,
+              y: -0.12895727420625547,
+              z: 0.5933086404370699
+            },
+            rotation: {
+              quaternion: {
+                W: 0.9890028788589879,
+                X: -0.0507354429843431,
+                Y: -0.13458187019694584,
+                Z: -0.034452004994036174
+              }
+            }
           },
           reprojectionErrors: [
             { x: 1, y: 1 },
             { x: 2, y: 1 },
             { x: 3, y: 1 }
           ],
-          includeObservationInCalibration: false,
+          cornersUsed: [true, true, false],
           snapshotName: "img0.png",
           snapshotData: { rows: 480, cols: 640, type: CvType.CV_8U, data: "" }
         }
       ],
+      calobjectSize: {
+        width: 10,
+        height: 10
+      },
+      calobjectSpacing: 0.0254,
+      lensModel: "opencv8",
       numSnapshots: 1,
-      meanErrors: [123.45]
+      meanErrors: [123.45],
+      numMissing: [0],
+      numOutliers: [1]
     }
   ],
   pipelineNicknames: ["Placeholder Pipeline"],

photon-core/src/main/java/org/photonvision/vision/calibration/BoardObservation.java

@@ -18,14 +18,23 @@
 package org.photonvision.vision.calibration;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import edu.wpi.first.math.geometry.Pose3d;
+import java.awt.Color;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.List;
 import org.jetbrains.annotations.Nullable;
+import org.opencv.core.Core;
+import org.opencv.core.Mat;
 import org.opencv.core.Point;
 import org.opencv.core.Point3;
+import org.opencv.core.Scalar;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+import org.photonvision.common.util.ColorHelper;
 
 // Ignore the previous calibration data that was stored in the json file.
 @JsonIgnoreProperties(ignoreUnknown = true)
@@ -47,8 +56,8 @@ public final class BoardObservation implements Cloneable {
     public Pose3d optimisedCameraToObject;
 
     // If we should use this observation when re-calculating camera calibration
-    @JsonProperty("includeObservationInCalibration")
-    public boolean includeObservationInCalibration;
+    @JsonProperty("cornersUsed")
+    public boolean[] cornersUsed;
 
     @JsonProperty("snapshotName")
     public String snapshotName;
@@ -63,16 +72,22 @@ public final class BoardObservation implements Cloneable {
             @JsonProperty("locationInImageSpace") List<Point> locationInImageSpace,
             @JsonProperty("reprojectionErrors") List<Point> reprojectionErrors,
             @JsonProperty("optimisedCameraToObject") Pose3d optimisedCameraToObject,
-            @JsonProperty("includeObservationInCalibration") boolean includeObservationInCalibration,
+            @JsonProperty("cornersUsed") boolean[] cornersUsed,
             @JsonProperty("snapshotName") String snapshotName,
             @JsonProperty("snapshotDataLocation") Path snapshotDataLocation) {
         this.locationInObjectSpace = locationInObjectSpace;
         this.locationInImageSpace = locationInImageSpace;
         this.reprojectionErrors = reprojectionErrors;
         this.optimisedCameraToObject = optimisedCameraToObject;
-        this.includeObservationInCalibration = includeObservationInCalibration;
         this.snapshotName = snapshotName;
         this.snapshotDataLocation = snapshotDataLocation;
+
+        // legacy migration -- we assume all points are inliers
+        if (cornersUsed == null) {
+            cornersUsed = new boolean[locationInObjectSpace.size()];
+            Arrays.fill(cornersUsed, true);
+        }
+        this.cornersUsed = cornersUsed;
     }
 
     @Override
@@ -85,8 +100,8 @@ public final class BoardObservation implements Cloneable {
                 + reprojectionErrors
                 + ", optimisedCameraToObject="
                 + optimisedCameraToObject
-                + ", includeObservationInCalibration="
-                + includeObservationInCalibration
+                + ", cornersUsed="
+                + cornersUsed
                 + ", snapshotName="
                 + snapshotName
                 + ", snapshotDataLocation="
@@ -103,4 +118,73 @@ public final class BoardObservation implements Cloneable {
             return null;
         }
     }
+
+    @JsonIgnore
+    /**
+     * Load the captured board image from disk. Allocates a new Mat, which the caller is responsible
+     * for releasing.
+     *
+     * @return The loaded image, or null if it could not be loaded.
+     */
+    public Mat loadImage() {
+        Mat img = Imgcodecs.imread(this.snapshotDataLocation.toString());
+        if (img == null || img.empty() || img.rows() == 0 || img.cols() == 0) {
+            return null;
+        }
+
+        return img;
+    }
+
+    /**
+     * Annotate the image with the detected corners, green for used, red for unused
+     *
+     * @return Annotated image, or null if the image could not be loaded. Caller is responsible for
+     *     releasing the Mat.
+     */
+    @JsonIgnore
+    public Mat annotateImage() {
+        var image = loadImage();
+
+        if (image == null) {
+            return null;
+        }
+
+        int thickness = Core.FILLED;
+        var diag = Math.hypot(image.width(), image.height());
+        int r = (int) Math.max(diag * 4.0 / 500.0, 3);
+        for (int i = 0; i < this.locationInImageSpace.size(); i++) {
+            var c = locationInImageSpace.get(i);
+
+            // -1, -1 means unused corner
+            if (c.x < 0 || c.y < 0) {
+                continue;
+            }
+
+            Scalar color;
+            if (cornersUsed[i]) {
+                color = ColorHelper.colorToScalar(Color.green);
+            } else {
+                color = ColorHelper.colorToScalar(Color.red);
+            }
+            Imgproc.circle(image, c, r, color, thickness);
+        }
+
+        return image;
+    }
+
+    /**
+     * Mean reprojection error for this observation, skipping corners marked as unused. The overall
+     * mean is calculated as the mean of each individual corner's reprojection error, or the distance
+     * in pixels between the observed and expected location.
+     *
+     * @return Mean reprojection error in pixels.
+     */
+    @JsonIgnore
+    double meanReprojectionError() {
+        return reprojectionErrors.stream()
+                .filter(pt -> cornersUsed[reprojectionErrors.indexOf(pt)])
+                .mapToDouble(pt -> Math.hypot(pt.x, pt.y))
+                .average()
+                .orElse(0);
+    }
 }

photon-core/src/main/java/org/photonvision/vision/calibration/CameraCalibrationCoefficients.java

@@ -217,7 +217,7 @@ public class CameraCalibrationCoefficients implements Releasable {
     }
 
     @JsonIgnore
-    public List<BoardObservation> getPerViewErrors() {
+    public List<BoardObservation> getObservations() {
         return observations;
     }
 

photon-core/src/main/java/org/photonvision/vision/calibration/UICameraCalibrationCoefficients.java

@@ -18,13 +18,19 @@
 package org.photonvision.vision.calibration;
 
 import java.util.List;
+import java.util.stream.IntStream;
 import org.opencv.core.Size;
 
 public class UICameraCalibrationCoefficients extends CameraCalibrationCoefficients {
     public int numSnapshots;
 
-    /** Immutable list of mean errors. */
     public List<Double> meanErrors;
+    public List<Integer> numMissing;
+    public List<Integer> numOutliers;
+
+    private static int countOutliers(BoardObservation obs) {
+        return (int) obs.locationInImageSpace.stream().filter(it -> it.x < 0 || it.y < 0).count();
+    }
 
     public UICameraCalibrationCoefficients(
             Size resolution,
@@ -47,14 +53,19 @@ public class UICameraCalibrationCoefficients extends CameraCalibrationCoefficien
                 lensmodel);
 
         this.numSnapshots = observations.size();
-        this.meanErrors =
+        this.meanErrors = observations.stream().map(BoardObservation::meanReprojectionError).toList();
+
+        this.numOutliers =
                 observations.stream()
                         .map(
-                                it2 ->
-                                        it2.reprojectionErrors.stream()
-                                                .mapToDouble(it -> Math.hypot(it.x, it.y))
-                                                .average()
-                                                .orElse(0))
+                                obs ->
+                                        IntStream.range(0, obs.cornersUsed.length)
+                                                        .filter(i -> !obs.cornersUsed[i])
+                                                        .map(i -> 1)
+                                                        .sum()
+                                                - countOutliers(obs))
                         .toList();
+        this.numMissing =
+                observations.stream().map(UICameraCalibrationCoefficients::countOutliers).toList();
     }
 }

photon-core/src/main/java/org/photonvision/vision/pipe/impl/Calibrate3dPipe.java

@@ -22,7 +22,6 @@ import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.stream.Collectors;
 import org.apache.commons.io.FileUtils;
 import org.opencv.calib3d.Calib3d;
 import org.opencv.core.*;
@@ -145,8 +144,8 @@ public class Calibrate3dPipe
         }
 
         // And delete rows depending on the level -- otherwise, level has no impact for opencv
-        List<Mat> objPoints = new ArrayList<>();
-        List<Mat> imgPoints = new ArrayList<>();
+        List<MatOfPoint3f> objPoints = new ArrayList<>();
+        List<MatOfPoint2f> imgPoints = new ArrayList<>();
         for (int i = 0; i < objPointsIn.size(); i++) {
             MatOfPoint3f objPtsOut = new MatOfPoint3f();
             MatOfPoint2f imgPtsOut = new MatOfPoint2f();
@@ -174,8 +173,8 @@ public class Calibrate3dPipe
             // imageSize from, other parameters are output Mats
 
             Calib3d.calibrateCameraExtended(
-                    objPoints,
-                    imgPoints,
+                    objPoints.stream().map(it -> (Mat) it).toList(),
+                    imgPoints.stream().map(it -> (Mat) it).toList(),
                     new Size(in.get(0).size.width, in.get(0).size.height),
                     cameraMatrix,
                     distortionCoefficients,
@@ -194,9 +193,29 @@ public class Calibrate3dPipe
         JsonMatOfDouble cameraMatrixMat = JsonMatOfDouble.fromMat(cameraMatrix);
         JsonMatOfDouble distortionCoefficientsMat = JsonMatOfDouble.fromMat(distortionCoefficients);
 
+        // Opencv is lame, so we can only assume all points are inliers
+        var inliners =
+                objPoints.stream()
+                        .map(
+                                it -> {
+                                    var array = new boolean[it.rows() * it.cols()];
+                                    Arrays.fill(array, true);
+                                    return array;
+                                })
+                        .toList();
+
         var observations =
                 createObservations(
-                        in, cameraMatrix, distortionCoefficients, rvecs, tvecs, null, imageSavePath);
+                        in,
+                        cameraMatrix,
+                        distortionCoefficients,
+                        rvecs,
+                        tvecs,
+                        inliners,
+                        new double[] {0, 0},
+                        objPoints,
+                        imgPoints,
+                        imageSavePath);
 
         cameraMatrix.release();
         distortionCoefficients.release();
@@ -266,11 +285,10 @@ public class Calibrate3dPipe
         JsonMatOfDouble distortionCoefficientsMat =
                 new JsonMatOfDouble(1, 8, CvType.CV_64FC1, Arrays.copyOfRange(result.intrinsics, 4, 12));
 
-        // Calculate optimized board poses manually. We get this for free from mrcal
-        // too, but that's not JNIed (yet)
+        // We get these from the JNI (retsult.optimizedPoses), but these are subtly different from the
+        // ones our code used to produce. To preserve consistency, continue to redo this math
         List<Mat> rvecs = new ArrayList<>();
         List<Mat> tvecs = new ArrayList<>();
-
         for (var o : in) {
             var rvec = new Mat();
             var tvec = new Mat();
@@ -309,6 +327,8 @@ public class Calibrate3dPipe
             tvecs.add(tvec);
         }
 
+        List<MatOfPoint3f> objPoints = in.stream().map(it -> it.objectPoints).toList();
+        List<MatOfPoint2f> imgPts = in.stream().map(it -> it.imagePoints).toList();
         List<BoardObservation> observations =
                 createObservations(
                         in,
@@ -316,7 +336,10 @@ public class Calibrate3dPipe
                         distortionCoefficientsMat.getAsMatOfDouble(),
                         rvecs,
                         tvecs,
+                        result.cornersUsed,
                         new double[] {result.warp_x, result.warp_y},
+                        objPoints,
+                        imgPts,
                         imageSavePath);
 
         rvecs.forEach(Mat::release);
@@ -339,13 +362,12 @@ public class Calibrate3dPipe
             MatOfDouble distortionCoefficients_,
             List<Mat> rvecs,
             List<Mat> tvecs,
+            List<boolean[]> cornersUsed,
             double[] calobject_warp,
+            List<MatOfPoint3f> objPoints,
+            List<MatOfPoint2f> imgPts,
             Path imageSavePath) {
-        List<Mat> objPoints = in.stream().map(it -> it.objectPoints).collect(Collectors.toList());
-        List<Mat> imgPts = in.stream().map(it -> it.imagePoints).collect(Collectors.toList());
-
         // Clear the calibration image folder of any old images before we save the new ones.
-
         try {
             FileUtils.cleanDirectory(imageSavePath.toFile());
         } catch (Exception e) {
@@ -355,11 +377,24 @@ public class Calibrate3dPipe
         // For each observation, calc reprojection error
         Mat jac_temp = new Mat();
         List<BoardObservation> observations = new ArrayList<>();
-        for (int i = 0; i < objPoints.size(); i++) {
+        for (int snapshotId = 0; snapshotId < objPoints.size(); snapshotId++) {
+            // Copy object points to a new mat to allow warp modification without affecting underlying
+            // data
             MatOfPoint3f i_objPtsNative = new MatOfPoint3f();
-            objPoints.get(i).copyTo(i_objPtsNative);
-            var i_objPts = i_objPtsNative.toList();
-            var i_imgPts = ((MatOfPoint2f) imgPts.get(i)).toList();
+            objPoints.get(snapshotId).copyTo(i_objPtsNative);
+
+            List<Point> i_imgPts = imgPts.get(snapshotId).toList();
+
+            if (i_objPtsNative.rows() != i_imgPts.size()) {
+                throw new RuntimeException(
+                        "Objpts size ("
+                                + i_objPtsNative.rows()
+                                + ") != imgpts size ("
+                                + i_imgPts.size()
+                                + ") for snapshot "
+                                + snapshotId
+                                + "!");
+            }
 
             // Apply warp, if set
             if (calobject_warp != null && calobject_warp.length == 2) {
@@ -384,12 +419,13 @@ public class Calibrate3dPipe
                 i_objPtsNative.fromArray(list);
             }
 
+            // Project distorted object points to image space
             var img_pts_reprojected = new MatOfPoint2f();
             try {
                 Calib3d.projectPoints(
                         i_objPtsNative,
-                        rvecs.get(i),
-                        tvecs.get(i),
+                        rvecs.get(snapshotId),
+                        tvecs.get(snapshotId),
                         cameraMatrix_,
                         distortionCoefficients_,
                         img_pts_reprojected,
@@ -399,22 +435,38 @@ public class Calibrate3dPipe
                 e.printStackTrace();
                 continue;
             }
-            var img_pts_reprojected_list = img_pts_reprojected.toList();
 
+            // Calculate reprojection error for each point
             var reprojectionError = new ArrayList<Point>();
+            var img_pts_reprojected_list = img_pts_reprojected.toList();
             for (int j = 0; j < img_pts_reprojected_list.size(); j++) {
+                // Outliers are not part of the calibration, so don't calculate error for them
+                if (!cornersUsed.get(snapshotId)[j]) {
+                    continue;
+                }
+
                 // error = (measured - expected)
                 var measured = img_pts_reprojected_list.get(j);
                 var expected = i_imgPts.get(j);
+
+                // Sanity check -- negative corners make no sense here
+                if (!(measured.x >= 0 && measured.y >= 0 && expected.x >= 0 && expected.y >= 0)) {
+                    throw new RuntimeException(
+                            "Negative corner in reprojection error calc! Measured: "
+                                    + measured
+                                    + ", expected: "
+                                    + expected);
+                }
+
                 var error = new Point(measured.x - expected.x, measured.y - expected.y);
                 reprojectionError.add(error);
             }
 
-            var camToBoard = MathUtils.opencvRTtoPose3d(rvecs.get(i), tvecs.get(i));
+            var camToBoard = MathUtils.opencvRTtoPose3d(rvecs.get(snapshotId), tvecs.get(snapshotId));
 
-            var inputImage = in.get(i).inputImage;
+            var inputImage = in.get(snapshotId).inputImage;
             Path image_path = null;
-            String snapshotName = "img" + i + ".png";
+            String snapshotName = "img" + snapshotId + ".png";
             if (inputImage != null) {
                 image_path = Paths.get(imageSavePath.toString(), snapshotName);
                 Imgcodecs.imwrite(image_path.toString(), inputImage);
@@ -422,7 +474,13 @@ public class Calibrate3dPipe
 
             observations.add(
                     new BoardObservation(
-                            i_objPts, i_imgPts, reprojectionError, camToBoard, true, snapshotName, image_path));
+                            i_objPtsNative.toList(),
+                            i_imgPts,
+                            reprojectionError,
+                            camToBoard,
+                            cornersUsed.get(snapshotId),
+                            snapshotName,
+                            image_path));
         }
         jac_temp.release();
 

photon-core/src/main/java/org/photonvision/vision/pipe/impl/FindBoardCornersPipe.java

@@ -287,7 +287,23 @@ public class FindBoardCornersPipe
             imgPoints.copyTo(outBoardCorners);
             objPoints.copyTo(objPts);
 
-            // Since ChaArUco can still detect without the whole board we need to send "fake" (all
+            // Mrcal wants our top-left corner at 0, 0. But charuco hands us the first corner at the first
+            // board intersection, which is inset a couple mm. Adjust such that the top-left corner is at
+            // 0,0
+            {
+                // don't trust any particular ordering
+                List<Point3> pointList = objPts.toList();
+                double minX = pointList.stream().mapToDouble(p -> p.x).min().orElse(0.0);
+                double minY = pointList.stream().mapToDouble(p -> p.y).min().orElse(0.0);
+
+                // Shift all object points so that the origin is at (0,0)
+                List<Point3> shiftedPoints =
+                        pointList.stream().map(p -> new Point3(p.x - minX, p.y - minY, p.z)).toList();
+
+                objPts.fromList(shiftedPoints);
+            }
+
+            // Since ChArUco can still detect without the whole board we need to send "fake" (all
             // values less than zero) points and then tell it to ignore that corner by setting the
             // corresponding level to -1. Calibrate3dPipe deals with piping this into the correct format
             // for each backend
@@ -298,12 +314,17 @@ public class FindBoardCornersPipe
                         new Point3[(this.params.boardHeight() - 1) * (this.params.boardWidth() - 1)];
                 levels = new float[(this.params.boardHeight() - 1) * (this.params.boardWidth() - 1)];
 
+                // cache
+                var outBoardCornersList = outBoardCorners.toList();
+                var outObjPtsList = objPts.toList();
+
                 for (int i = 0; i < detectedIds.total(); i++) {
                     int id = (int) detectedIds.get(i, 0)[0];
-                    boardCorners[id] = outBoardCorners.toList().get(i);
-                    objectPoints[id] = objPts.toList().get(i);
+                    boardCorners[id] = outBoardCornersList.get(i);
+                    objectPoints[id] = outObjPtsList.get(i);
                     levels[id] = 1.0f;
                 }
+
                 for (int i = 0; i < boardCorners.length; i++) {
                     if (boardCorners[i] == null) {
                         boardCorners[i] = new Point(-1, -1);
@@ -320,7 +341,6 @@ public class FindBoardCornersPipe
             objPoints.release();
             detectedCorners.release();
             detectedIds.release();
-
         } else { // If not ChArUco then do chessboard
             // Reduce the image size to be much more manageable
             // Note that opencv will copy the frame if no resize is requested; we can skip

photon-core/src/test/java/org/photonvision/vision/pipeline/Calibrate3dPipeTest.java

@@ -279,6 +279,18 @@ public class Calibrate3dPipeTest {
         System.out.println("Camera Intrinsics: " + cal.cameraIntrinsics.toString());
         System.out.println("Dist Coeffs: " + cal.distCoeffs.toString());
 
+        // calculate RMS error
+        double totalSquaredError = 0.0;
+        long totalPoints = 0;
+        for (var obs : cal.getObservations()) {
+            double sumErrorSq =
+                    obs.reprojectionErrors.stream().mapToDouble(d -> d.x * d.x + d.y * d.y).sum();
+            totalSquaredError += sumErrorSq;
+            totalPoints += obs.reprojectionErrors.size();
+        }
+        double rmsError = Math.sqrt(totalSquaredError / totalPoints);
+        System.out.println("RMS Reprojection Error: " + rmsError);
+
         // Confirm we didn't get leaky on our mat usage
         // assertEquals(startMatCount, CVMat.getMatCount()); // TODO Figure out why this
         // doesn't

photon-server/src/main/java/org/photonvision/Main.java

@@ -187,6 +187,15 @@ public class Main {
     }
 
     public static void main(String[] args) {
+        var logLevel = printDebugLogs ? LogLevel.TRACE : LogLevel.DEBUG;
+        Logger.setLevel(LogGroup.Camera, logLevel);
+        Logger.setLevel(LogGroup.WebServer, logLevel);
+        Logger.setLevel(LogGroup.VisionModule, logLevel);
+        Logger.setLevel(LogGroup.Data, logLevel);
+        Logger.setLevel(LogGroup.Config, logLevel);
+        Logger.setLevel(LogGroup.General, logLevel);
+        logger.info("Logging initialized in debug mode.");
+
         logger.info(
                 "Starting PhotonVision version "
                         + PhotonVersion.versionString
@@ -257,15 +266,6 @@ public class Main {
         CVMat.enablePrint(false);
         PipelineProfiler.enablePrint(false);
 
-        var logLevel = printDebugLogs ? LogLevel.TRACE : LogLevel.DEBUG;
-        Logger.setLevel(LogGroup.Camera, logLevel);
-        Logger.setLevel(LogGroup.WebServer, logLevel);
-        Logger.setLevel(LogGroup.VisionModule, logLevel);
-        Logger.setLevel(LogGroup.Data, logLevel);
-        Logger.setLevel(LogGroup.Config, logLevel);
-        Logger.setLevel(LogGroup.General, logLevel);
-        logger.info("Logging initialized in debug mode.");
-
         // Add Linux kernel log->Photon logger
         KernelLogLogger.getInstance();
 

photon-server/src/main/java/org/photonvision/server/RequestHandler.java

@@ -32,7 +32,6 @@ import java.util.LinkedList;
 import java.util.Optional;
 import javax.imageio.ImageIO;
 import org.apache.commons.io.FileUtils;
-import org.opencv.core.Mat;
 import org.opencv.core.MatOfByte;
 import org.opencv.core.MatOfInt;
 import org.opencv.core.Size;
@@ -525,7 +524,7 @@ public class RequestHandler {
     public static void onDataCalibrationImportRequest(Context ctx) {
         try {
             DataCalibrationImportRequest request =
-                    kObjectMapper.readValue(ctx.body(), DataCalibrationImportRequest.class);
+                    kObjectMapper.readValue(ctx.req().getInputStream(), DataCalibrationImportRequest.class);
 
             var uploadCalibrationEvent =
                     new IncomingWebSocketEvent<>(
@@ -1000,6 +999,41 @@ public class RequestHandler {
         ctx.status(204);
     }
 
+    /**
+     * Get the calibration JSON for a specific observation. Excludes camera image data
+     *
+     * <p>This is excluded from UICalibrationCoefficients by default to save bandwidth on large
+     * calibrations
+     */
+    public static void onCalibrationJsonRequest(Context ctx) {
+        String cameraUniqueName = ctx.queryParam("cameraUniqueName");
+        var width = Integer.parseInt(ctx.queryParam("width"));
+        var height = Integer.parseInt(ctx.queryParam("height"));
+
+        var module = VisionSourceManager.getInstance().vmm.getModule(cameraUniqueName);
+        if (module == null) {
+            ctx.status(404);
+            return;
+        }
+
+        CameraCalibrationCoefficients calList =
+                module.getStateAsCameraConfig().calibrations.stream()
+                        .filter(
+                                it ->
+                                        Math.abs(it.unrotatedImageSize.width - width) < 1e-4
+                                                && Math.abs(it.unrotatedImageSize.height - height) < 1e-4)
+                        .findFirst()
+                        .orElse(null);
+
+        if (calList == null) {
+            ctx.status(404);
+            return;
+        }
+
+        ctx.json(calList);
+        ctx.status(200);
+    }
+
     private record CalibrationRemoveRequest(int width, int height, String cameraUniqueName) {}
 
     public static void onCalibrationRemoveRequest(Context ctx) {
@@ -1065,28 +1099,18 @@ public class RequestHandler {
             return;
         }
 
-        // encode as jpeg to save even more space. reduces size of a 1280p image from
-        // 300k to 25k
+        // encode as jpeg to save even more space. reduces size of a 1280p image from 300k to 25k
+        var mat = calList.observations.get(observationIdx).annotateImage();
+        if (mat == null) {
+            ctx.status(404);
+            return;
+        }
+
         var jpegBytes = new MatOfByte();
-        Mat img = null;
-        try {
-            img =
-                    Imgcodecs.imread(
-                            calList.observations.get(observationIdx).snapshotDataLocation.toString());
-        } catch (Exception e) {
-            ctx.status(500);
-            ctx.result("Unable to read calibration image");
-            return;
-        }
-        if (img == null || img.empty()) {
-            ctx.status(500);
-            ctx.result("Unable to read calibration image");
-            return;
-        }
-
-        Imgcodecs.imencode(".jpg", img, jpegBytes, new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 60));
-
+        Imgcodecs.imencode(".jpg", mat, jpegBytes, new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 60));
         ctx.result(jpegBytes.toArray());
+
+        mat.release();
         jpegBytes.release();
 
         ctx.status(200);

photon-server/src/main/java/org/photonvision/server/Server.java

@@ -68,7 +68,6 @@ public class Server {
                                                             it.anyHost();
                                                         });
                                             }));
-
                             javalinConfig.requestLogger.http(
                                     (ctx, ms) -> {
                                         StringJoiner joiner =
@@ -129,6 +128,7 @@ public class Server {
         app.post("/api/settings/camera", RequestHandler::onCameraSettingsRequest);
         app.post("/api/settings/camera/setNickname", RequestHandler::onCameraNicknameChangeRequest);
         app.get("/api/settings/camera/getCalibImages", RequestHandler::onCameraCalibImagesRequest);
+        app.get("/api/settings/camera/getCalibration", RequestHandler::onCalibrationJsonRequest);
 
         // Utilities
         app.post("/api/utils/offlineUpdate", RequestHandler::onOfflineUpdateRequest);