Backport maven changes

* Sort object detection results and reduce code dup.

* Filter objdet results by ratio and area

* Address code review

---------

Co-authored-by: Mohammad Durrani <46766905+mdurrani808@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,7 +18,7 @@ Steps to reproduce the behavior:
 4. See error
 
 **Screenshots / Videos**
-If applicable, add screenshots to help explain your problem. Additionally, provide journalctl logs and settings zip export.
+If applicable, add screenshots to help explain your problem. Additionally, provide journalctl logs and settings zip export. If your issue is regarding the web dashboard, please provide screenshots and the output of the browser console.
 
 **Platform:**
  - Hardware Platform (ex. Raspberry Pi 4, Windows x64):

.github/workflows/build.yml

@@ -2,124 +2,17 @@ name: Build
 
 on:
   push:
-    branches: [ master ]
+    branches: 
+      - master
     tags:
       - 'v*'
   pull_request:
     branches: [ master ]
 
 jobs:
-  build-client:
-    name: "PhotonClient Build"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Build Production Client
-        run: npm run build
-      - uses: actions/upload-artifact@master
-        with:
-          name: built-client
-          path: photon-client/dist/
-  build-examples:
-    name: "Build Examples"
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      # Need to publish to maven local first, so that C++ sim can pick it up
-      # Still haven't figured out how to make the vendordep file be copied before trying to build examples
-      - name: Publish photonlib to maven local
-        run: |
-          chmod +x gradlew
-          ./gradlew publishtomavenlocal -x check
-      - name: Build Java examples
-        working-directory: photonlib-java-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-      - name: Build C++ examples
-        working-directory: photonlib-cpp-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-  build-gradle:
-    name: "Gradle Build"
-    runs-on: ubuntu-22.04
-    steps:
-      # Checkout code.
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Gradle Build
-        run: |
-          chmod +x gradlew
-          ./gradlew photon-server:build photon-lib:build -x check --max-workers 2
-      - name: Gradle Tests
-        run: ./gradlew testHeadless -i --max-workers 1 --stacktrace
-      - name: Gradle Coverage
-        run: ./gradlew jacocoTestReport --max-workers 1
-      - name: Publish Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-server/build/reports/jacoco/test/jacocoTestReport.xml
-      - name: Publish Core Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-core/build/reports/jacoco/test/jacocoTestReport.xml
-  build-offline-docs:
-    name: "Build Offline Docs"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: 'PhotonVision/photonvision-docs.git'
-          ref: master
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install sphinx sphinx_rtd_theme sphinx-tabs sphinxext-opengraph doc8
-          pip install -r requirements.txt
-      - name: Build the docs
-        run: |
-          make html
-      - uses: actions/upload-artifact@master
-        with:
-          name: built-docs
-          path: build/html
   build-photonlib-host:
     env:
-      MACOSX_DEPLOYMENT_TARGET: 11
+      MACOSX_DEPLOYMENT_TARGET: 12
     strategy:
       fail-fast: false
       matrix:
@@ -136,11 +29,11 @@ jobs:
     name: "Photonlib - Build Host - ${{ matrix.artifact-name }}"
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Java 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 17
           distribution: temurin
@@ -169,7 +62,7 @@ jobs:
     container: ${{ matrix.container }}
     name: "Photonlib - Build Docker - ${{ matrix.artifact-name }}"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Config Git
@@ -182,143 +75,7 @@ jobs:
       - name: Publish
         run: |
           chmod +x gradlew
-          ./gradlew photon-lib:publish
+          ./gradlew photon-lib:publish photon-targeting:publish
         env:
           ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }}
         if: github.event_name == 'push'
-  build-package:
-    needs: [build-client, build-gradle, build-offline-docs]
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: windows-latest
-            artifact-name: Win64
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOS
-            architecture: x64
-            arch-override: none
-          - os: ubuntu-latest
-            artifact-name: Linux
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOSArm
-            architecture: x64
-            arch-override: macarm64
-          - os: ubuntu-latest
-            artifact-name: LinuxArm32
-            architecture: x64
-            arch-override: linuxarm32
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            architecture: x64
-            arch-override: linuxarm64
-
-    runs-on: ${{ matrix.os }}
-    name: "Build fat JAR - ${{ matrix.artifact-name }}"
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Install Java 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          rm -rf photon-server/src/main/resources/web/*
-          mkdir -p photon-server/src/main/resources/web/docs
-        if: ${{ (matrix.os) != 'windows-latest' }}
-      - run: |
-          del photon-server\src\main\resources\web\*.*
-          mkdir photon-server\src\main\resources\web\docs
-        if: ${{ (matrix.os) == 'windows-latest' }}
-      - uses: actions/download-artifact@v3
-        with:
-          name: built-client
-          path: photon-server/src/main/resources/web/
-      - uses: actions/download-artifact@v3
-        with:
-          name: built-docs
-          path: photon-server/src/main/resources/web/docs
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2 -PArchOverride=${{ matrix.arch-override }}
-        if: ${{ (matrix.arch-override != 'none') }}
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2
-        if: ${{ (matrix.arch-override == 'none') }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: jar-${{ matrix.artifact-name }}
-          path: photon-server/build/libs
-  build-image:
-    needs: [build-package]
-
-    if: ${{ github.event_name != 'pull_request' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: RaspberryPi
-            image_url: https://api.github.com/repos/photonvision/photon-pi-gen/releases/tags/v2023.1.3_arm64
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: limelight2
-            image_url: https://api.github.com/repos/photonvision/photon-pi-gen/releases/tags/v2023.2.2_limelight-arm64
-
-    runs-on: ${{ matrix.os }}
-    name: "Build image - ${{ matrix.image_url }}"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/download-artifact@v2
-        with:
-          name: jar-${{ matrix.artifact-name }}
-      - name: Generate image
-        run: |
-          chmod +x scripts/generatePiImage.sh
-          ./scripts/generatePiImage.sh ${{ matrix.image_url }} ${{ matrix.image_suffix }}
-      - uses: actions/upload-artifact@v3
-        name: Upload image
-        with:
-          name: image-${{ matrix.image_suffix }}
-          path: photonvision*.xz
-  release:
-    needs: [build-package, build-image]
-    runs-on: ubuntu-22.04
-    steps:
-      # Download literally every single artifact. This also downloads client and docs,
-      # but the filtering below won't pick these up (I hope)
-      - uses: actions/download-artifact@v2
-      - run: find
-      # Push to dev release
-      - uses: pyTooling/Actions/releaser@r0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          tag: 'Dev'
-          rm: true
-          files: |
-            **/*.xz
-            **/*.jar
-        if: github.event_name == 'push'
-      # Upload all jars and xz archives
-      - uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            **/*.xz
-            **/*.jar
-        if: startsWith(github.ref, 'refs/tags/v')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lint-format.yml

@@ -1,88 +0,0 @@
-name: Lint and Format
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
-  pull_request:
-    branches: [ master ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  wpiformat:
-    name: "wpiformat"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Install wpiformat
-        run: pip3 install wpiformat
-      - name: Run
-        run: wpiformat
-      - name: Check output
-        run: git --no-pager diff --exit-code HEAD
-      - name: Generate diff
-        run: git diff HEAD > wpiformat-fixes.patch
-        if: ${{ failure() }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: wpiformat fixes
-          path: wpiformat-fixes.patch
-        if: ${{ failure() }}
-  javaformat:
-    name: "Java Formatting"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          chmod +x gradlew
-          ./gradlew spotlessCheck
-
-  client-lint-format:
-    name: "PhotonClient Lint and Formatting"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Check Linting
-        run: npm run lint-ci
-      - name: Check Formatting
-        run: npm run format-ci
-  server-index:
-    name: "Check server index.html not changed"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Check index.html not changed
-        run: git --no-pager diff --exit-code origin/master photon-server/src/main/resources/web/index.html

.gitignore

@@ -162,3 +162,5 @@ photonlib-cpp-examples/*/networktables.json.bck
 photonlib-java-examples/*/networktables.json.bck
 *.sqlite
 photon-server/src/main/resources/web/index.html
+
+venv

.styleguide

@@ -18,6 +18,7 @@ modifiableFileExclude {
   \.dll$
   \.webp$
   \.ico$
+  \.rknn$
   gradlew
 }
 

README.md

@@ -59,6 +59,16 @@ To run them, use the commands listed below. Photonlib must first be published to
 ~/photonvision/photonlib-cpp-examples$ ./gradlew <example-name>:simulateNative
 ```
 
+## Out-of-Source Dependencies
+
+PhotonVision uses the following additonal out-of-source repositories for building code.
+
+- Base system images for Raspberry Pi & Orange Pi: https://github.com/PhotonVision/photon-image-modifier
+- C++ driver for Raspberry Pi CSI cameras: https://github.com/PhotonVision/photon-libcamera-gl-driver
+- JNI code for [mrcal](https://mrcal.secretsauce.net/): https://github.com/PhotonVision/mrcal-java
+- Custom build of OpenCV with GStreamer/Protobuf/other custom flags: https://github.com/PhotonVision/thirdparty-opencv
+- JNI code for aruco-nano: https://github.com/PhotonVision/aruconano-jni
+
 
 ## Acknowledgments
 PhotonVision was forked from [Chameleon Vision](https://github.com/Chameleon-Vision/chameleon-vision/). Thank you to everyone who worked on the original project.
@@ -82,3 +92,18 @@ Our meeting notes can be found in the wiki section of this repository.
 
 * [2020 Meeting Notes](https://github.com/PhotonVision/photonvision/wiki/2020-Meeting-Notes)
 * [2021 Meeting Notes](https://github.com/PhotonVision/photonvision/wiki/2021-Meeting-Notes)
+
+## Additional packages
+
+For now, using mrcal requires installing these additional packages on Linux systems:
+
+```
+sudo apt install libcholmod3 liblapack3 libsuitesparseconfig5
+```
+
+## Documentation
+
+- Our main documentation page: [docs.photonvision.org](https://docs.photonvision.org)
+- Photon UI demo: [demo.photonvision.org](https://demo.photonvision.org) (or [manual link](https://photonvision.github.io/photonvision/built-client/))
+- Javadocs: [javadocs.photonvision.org](https://javadocs.photonvision.org) (or [manual link](https://photonvision.github.io/photonvision/built-docs/javadoc/))
+- C++ Doxygen  [cppdocs.photonvision.org](https://cppdocs.photonvision.org) (or [manual link](https://photonvision.github.io/photonvision/built-docs/doxygen/html/))

build.gradle

@@ -1,16 +1,21 @@
+import edu.wpi.first.toolchain.*
+
 plugins {
-    id "com.diffplug.spotless" version "6.22.0"
+    id "com.diffplug.spotless" version "6.24.0"
-    id "edu.wpi.first.NativeUtils" version "2024.2.0" apply false
+    id "edu.wpi.first.NativeUtils" version "2024.6.1" apply false
     id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2"
-    id "edu.wpi.first.GradleRIO" version "2024.1.1-beta-3"
+    id "edu.wpi.first.GradleRIO" version "2024.2.1"
     id 'edu.wpi.first.WpilibTools' version '1.3.0'
+    id 'com.google.protobuf' version '0.9.4' apply false
 }
 
 allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
+        maven { url = "https://maven.photonvision.org/releases" }
+        maven { url = "https://maven.photonvision.org/snapshots" }
+        maven { url = "https://jogamp.org/deployment/maven/" }
     }
     wpilibRepositories.addAllReleaseRepositories(it)
     wpilibRepositories.addAllDevelopmentRepositories(it)
@@ -20,32 +25,38 @@ allprojects {
 apply from: "versioningHelper.gradle"
 
 ext {
-    wpilibVersion = "2024.1.1-beta-3"
+    wpilibVersion = "2024.2.1"
+    wpimathVersion = wpilibVersion
     openCVversion = "4.8.0-2"
     joglVersion = "2.4.0-rc-20200307"
     javalinVersion = "5.6.2"
+    photonGlDriverLibVersion = "dev-v2023.1.0-9-g75fc678"
+    rknnVersion = "dev-v2024.0.0-30-g001b5ec"
     frcYear = "2024"
+    mrcalVersion = "dev-v2024.0.0-7-gc976aaa";
+
 
     pubVersion = versionString
     isDev = pubVersion.startsWith("dev")
 
     // A list, for legacy reasons, with only the current platform contained
-    String nativeName = wpilibTools.platformMapper.currentPlatform.platformName;
+    wpilibNativeName = wpilibTools.platformMapper.currentPlatform.platformName;
-    if (nativeName == "linuxx64") nativeName = "linuxx86-64";
+    def nativeName = wpilibNativeName
-    if (nativeName == "winx64") nativeName = "windowsx86-64";
+    if (wpilibNativeName == "linuxx64") nativeName = "linuxx86-64";
-    if (nativeName == "macx64") nativeName = "osxx86-64";
+    if (wpilibNativeName == "winx64") nativeName = "windowsx86-64";
-    if (nativeName == "macarm64") nativeName = "osxarm64";
+    if (wpilibNativeName == "macx64") nativeName = "osxx86-64";
+    if (wpilibNativeName == "macarm64") nativeName = "osxarm64";
     jniPlatform = nativeName
-    println("Building for platform " + jniPlatform)
+
+    println("Building for platform " + jniPlatform + " wpilib: " + wpilibNativeName)
+    println("Using Wpilib: " + wpilibVersion)
+    println("Using OpenCV: " + openCVversion)
+
+
+    photonMavenURL = 'https://maven.photonvision.org/' + (isDev ? 'snapshots' : 'releases');
+    println("Publishing Photonlib to " + photonMavenURL)
 }
 
-wpilibTools.deps.wpilibVersion = wpilibVersion
-
-// Tell gradlerio what version of things to use (that we care about)
-// See: https://github.com/wpilibsuite/GradleRIO/blob/main/src/main/java/edu/wpi/first/gradlerio/wpi/WPIVersionsExtension.java
-wpi.getVersions().getOpencvVersion().convention(openCVversion);
-wpi.getVersions().getWpilibVersion().convention(wpilibVersion);
-
 spotless {
     java {
         target fileTree('.') {
@@ -94,3 +105,7 @@ spotless {
 wrapper {
     gradleVersion '8.4'
 }
+
+ext.getCurrentArch = {
+    return NativePlatforms.desktop
+}

devTools/calibrationUtils.py

@@ -0,0 +1,255 @@
+import argparse
+import base64
+from dataclasses import dataclass
+import json
+import os
+from typing import Union
+import cv2
+import numpy as np
+import mrcal
+from wpimath.geometry import Quaternion as _Quat
+
+
+@dataclass
+class Size:
+    width: int
+    height: int
+
+
+@dataclass
+class JsonMatOfDoubles:
+    rows: int
+    cols: int
+    type: int
+    data: list[float]
+
+
+@dataclass
+class JsonMat:
+    rows: int
+    cols: int
+    type: int
+    data: str  # Base64-encoded PNG data
+
+
+@dataclass
+class Point2:
+    x: float
+    y: float
+
+
+@dataclass
+class Translation3d:
+    x: float
+    y: float
+    z: float
+
+
+@dataclass
+class Quaternion:
+    X: float
+    Y: float
+    Z: float
+    W: float
+
+
+@dataclass
+class Rotation3d:
+    quaternion: Quaternion
+
+
+@dataclass
+class Pose3d:
+    translation: Translation3d
+    rotation: Rotation3d
+
+
+@dataclass
+class Point3:
+    x: float
+    y: float
+    z: float
+
+
+@dataclass
+class Observation:
+    # Expected feature 3d location in the camera frame
+    locationInObjectSpace: list[Point3]
+    # Observed location in pixel space
+    locationInImageSpace: list[Point2]
+    # (measured location in pixels) - (expected from FK)
+    reprojectionErrors: list[Point2]
+    # Solver optimized board poses
+    optimisedCameraToObject: Pose3d
+    # If we should use this observation when re-calculating camera calibration
+    includeObservationInCalibration: bool
+    snapshotName: str
+    # The actual image the snapshot is from
+    snapshotData: JsonMat
+
+
+@dataclass
+class CameraCalibration:
+    resolution: Size
+    cameraIntrinsics: JsonMatOfDoubles
+    distCoeffs: JsonMatOfDoubles
+    observations: list[Observation]
+    calobjectWarp: list[float]
+    calobjectSize: Size
+    calobjectSpacing: float
+
+
+def __convert_cal_to_mrcal_cameramodel(
+    cal: CameraCalibration,
+) -> mrcal.cameramodel | None:
+    if len(cal.distCoeffs.data) == 5:
+        model = "LENSMODEL_OPENCV5"
+    elif len(cal.distCoeffs.data) == 8:
+        model = "LENSMODEL_OPENCV8"
+    else:
+        print("Unknown camera model? giving up")
+        return None
+
+    def opencv_to_mrcal_intrinsics(ocv):
+        return [ocv[0], ocv[4], ocv[2], ocv[5]]
+
+    def pose_to_rt(pose: Pose3d):
+        r = _Quat(
+            w=pose.rotation.quaternion.W,
+            x=pose.rotation.quaternion.X,
+            y=pose.rotation.quaternion.Y,
+            z=pose.rotation.quaternion.Z,
+        ).toRotationVector()
+        t = [
+            pose.translation.x,
+            pose.translation.y,
+            pose.translation.z,
+        ]
+        return np.concatenate((r, t))
+
+    imagersize = (cal.resolution.width, cal.resolution.height)
+
+    # Always weight=1 for Photon data
+    WEIGHT = 1
+    observations_board = np.array(
+        [
+            # note that we expect row-major observations here. I think this holds
+            np.array(
+                list(map(lambda it: [it.x, it.y, WEIGHT], o.locationInImageSpace))
+            ).reshape((cal.calobjectSize.width, cal.calobjectSize.height, 3))
+            for o in cal.observations
+        ]
+    )
+
+    optimization_inputs = {
+        "intrinsics": np.array(
+            [
+                opencv_to_mrcal_intrinsics(cal.cameraIntrinsics.data)
+                + cal.distCoeffs.data
+            ],
+            dtype=np.float64,
+        ),
+        "extrinsics_rt_fromref": np.zeros((0, 6), dtype=np.float64),
+        "frames_rt_toref": np.array(
+            [pose_to_rt(o.optimisedCameraToObject) for o in cal.observations]
+        ),
+        "points": None,
+        "observations_board": observations_board,
+        "indices_frame_camintrinsics_camextrinsics": np.array(
+            [[i, 0, -1] for i in range(len(cal.observations))], dtype=np.int32
+        ),
+        "observations_point": None,
+        "indices_point_camintrinsics_camextrinsics": None,
+        "lensmodel": model,
+        "imagersizes": np.array([imagersize], dtype=np.int32),
+        "calobject_warp": np.array(cal.calobjectWarp)
+        if len(cal.calobjectWarp) > 0
+        else None,
+        # We always do all the things
+        "do_optimize_intrinsics_core": True,
+        "do_optimize_intrinsics_distortions": True,
+        "do_optimize_extrinsics": True,
+        "do_optimize_frames": True,
+        "do_optimize_calobject_warp": len(cal.calobjectWarp) > 0,
+        "do_apply_outlier_rejection": True,
+        "do_apply_regularization": True,
+        "verbose": False,
+        "calibration_object_spacing": cal.calobjectSpacing,
+        "imagepaths": np.array([it.snapshotName for it in cal.observations]),
+    }
+
+    return mrcal.cameramodel(
+        optimization_inputs=optimization_inputs,
+        icam_intrinsics=0,
+    )
+
+
+def convert_photon_to_mrcal(photon_cal_json_path: str, output_folder: str):
+    """
+    Unpack a Photon calibration JSON (eg, photon_calibration_Microsoft_LifeCam_HD-3000_800x600.json) into
+    the output_folder directory with images and corners.vnl file for use with mrcal.
+    """
+    with open(photon_cal_json_path, "r") as cal_json:
+        # Convert to nested objects instead of nameddicts on json-loads
+        class Generic:
+            @classmethod
+            def from_dict(cls, dict):
+                obj = cls()
+                obj.__dict__.update(dict)
+                return obj
+
+        camera_cal_data: CameraCalibration = json.loads(
+            cal_json.read(), object_hook=Generic.from_dict
+        )
+
+        # Create output_folder if not exists
+        if not os.path.exists(output_folder):
+            os.makedirs(output_folder)
+
+        # Decode each image and save it as a png
+        for obs in camera_cal_data.observations:
+            image = obs.snapshotData.data
+            decoded_data = base64.b64decode(image)
+            np_data = np.frombuffer(decoded_data, np.uint8)
+            img = cv2.imdecode(np_data, cv2.IMREAD_UNCHANGED)
+            cv2.imwrite(f"{output_folder}/{obs.snapshotName}", img)
+
+        # And create a VNL file for use with mrcal
+        with open(f"{output_folder}/corners.vnl", "w+") as vnl_file:
+            vnl_file.write("# filename x y level\n")
+
+            for obs in camera_cal_data.observations:
+                for corner in obs.locationInImageSpace:
+                    # Always level zero
+                    vnl_file.write(f"{obs.snapshotName} {corner.x} {corner.y} 0\n")
+
+            vnl_file.flush()
+
+        mrcal_model = __convert_cal_to_mrcal_cameramodel(camera_cal_data)
+
+        with open(f"{output_folder}/camera-0.cameramodel", "w+") as mrcal_file:
+            mrcal_model.write(
+                mrcal_file,
+                note="Generated from PhotonVision calibration file: "
+                + photon_cal_json_path
+                + "\nCalobject_warp (m): "
+                + str(camera_cal_data.calobjectWarp),
+            )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert Photon calibration JSON for use with mrcal"
+    )
+    parser.add_argument("input", type=str, help="Path to Photon calibration JSON file")
+    parser.add_argument(
+        "output_folder", type=str, help="Output folder for mrcal VNL file + images"
+    )
+
+    args = parser.parse_args()
+
+    convert_photon_to_mrcal(args.input, args.output_folder)
+
+
+if __name__ == "__main__":
+    main()

docs/build.gradle

@@ -0,0 +1,287 @@
+// From allwpilib/docs. Licensed under the WPILib BSD License
+
+plugins {
+    id 'java'
+    id "org.ysb33r.doxygen" version "0.7.0"
+}
+
+
+evaluationDependsOn ':photon-targeting'
+evaluationDependsOn ':photon-core'
+evaluationDependsOn ':photon-server'
+evaluationDependsOn ':photon-lib'
+
+
+def baseArtifactIdCpp = 'documentation'
+def artifactGroupIdCpp = 'org.photonvision.wpilibc'
+def zipBaseNameCpp = '_GROUP_org.photonvision_cpp_ID_documentation_CLS'
+
+def baseArtifactIdJava = 'documentation'
+def artifactGroupIdJava = 'org.photonvision.wpilibj'
+def zipBaseNameJava = '_GROUP_org.photonvision_java_ID_documentation_CLS'
+
+def outputsFolder = file("$project.buildDir/outputs")
+
+def cppProjectZips = []
+def cppIncludeRoots = []
+
+cppProjectZips.add(project(':photon-lib').cppHeadersZip)
+cppProjectZips.add(project(':photon-targeting').cppHeadersZip)
+
+doxygen {
+    // Doxygen binaries are only provided for x86_64 platforms
+    // Other platforms will need to provide doxygen via their system
+    // See below maven and https://doxygen.nl/download.html for provided binaries
+
+    String arch = System.getProperty("os.arch");
+    if (arch.equals("x86_64") || arch.equals("amd64")) {
+        executables {
+            doxygen version : '1.9.4',
+            baseURI : 'https://frcmaven.wpi.edu/artifactory/generic-release-mirror/doxygen'
+        }
+    }
+}
+
+doxygen {
+    generate_html true
+    html_extra_stylesheet 'theme.css'
+
+    cppProjectZips.each {
+        dependsOn it
+        source it.source
+        it.ext.includeDirs.each {
+            cppIncludeRoots.add(it.absolutePath)
+        }
+    }
+    cppIncludeRoots << '../ntcore/build/generated/main/native/include/'
+
+    if (project.hasProperty('docWarningsAsErrors')) {
+        // Eigen
+        exclude 'Eigen/**'
+        exclude 'unsupported/**'
+
+        // LLVM
+        exclude 'wpi/AlignOf.h'
+        exclude 'wpi/Casting.h'
+        exclude 'wpi/Chrono.h'
+        exclude 'wpi/Compiler.h'
+        exclude 'wpi/ConvertUTF.h'
+        exclude 'wpi/DenseMap.h'
+        exclude 'wpi/DenseMapInfo.h'
+        exclude 'wpi/Endian.h'
+        exclude 'wpi/EpochTracker.h'
+        exclude 'wpi/Errc.h'
+        exclude 'wpi/Errno.h'
+        exclude 'wpi/ErrorHandling.h'
+        exclude 'wpi/bit.h'
+        exclude 'wpi/fs.h'
+        exclude 'wpi/FunctionExtras.h'
+        exclude 'wpi/function_ref.h'
+        exclude 'wpi/Hashing.h'
+        exclude 'wpi/iterator.h'
+        exclude 'wpi/iterator_range.h'
+        exclude 'wpi/ManagedStatic.h'
+        exclude 'wpi/MapVector.h'
+        exclude 'wpi/MathExtras.h'
+        exclude 'wpi/MemAlloc.h'
+        exclude 'wpi/PointerIntPair.h'
+        exclude 'wpi/PointerLikeTypeTraits.h'
+        exclude 'wpi/PointerUnion.h'
+        exclude 'wpi/raw_os_ostream.h'
+        exclude 'wpi/raw_ostream.h'
+        exclude 'wpi/SmallPtrSet.h'
+        exclude 'wpi/SmallSet.h'
+        exclude 'wpi/SmallString.h'
+        exclude 'wpi/SmallVector.h'
+        exclude 'wpi/StringExtras.h'
+        exclude 'wpi/StringMap.h'
+        exclude 'wpi/SwapByteOrder.h'
+        exclude 'wpi/type_traits.h'
+        exclude 'wpi/VersionTuple.h'
+        exclude 'wpi/WindowsError.h'
+
+        // fmtlib
+        exclude 'fmt/**'
+
+        // libuv
+        exclude 'uv.h'
+        exclude 'uv/**'
+        exclude 'wpinet/uv/**'
+
+        // json
+        exclude 'wpi/adl_serializer.h'
+        exclude 'wpi/byte_container_with_subtype.h'
+        exclude 'wpi/detail/**'
+        exclude 'wpi/json.h'
+        exclude 'wpi/json_fwd.h'
+        exclude 'wpi/ordered_map.h'
+        exclude 'wpi/thirdparty/**'
+
+        // memory
+        exclude 'wpi/memory/**'
+
+        // mpack
+        exclude 'wpi/mpack.h'
+
+        // units
+        exclude 'units/**'
+    }
+
+    //TODO: building memory docs causes search to break
+    exclude 'wpi/memory/**'
+
+    exclude '*.pb.h'
+
+    // Save space by excluding protobuf and eigen
+    exclude 'Eigen/**'
+    exclude 'google/protobuf/**'
+
+    aliases 'effects=\\par <i>Effects:</i>^^',
+            'notes=\\par <i>Notes:</i>^^',
+            'requires=\\par <i>Requires:</i>^^',
+            'requiredbe=\\par <i>Required Behavior:</i>^^',
+            'concept{2}=<a href=\"md_doc_concepts.html#\1\">\2</a>',
+            'defaultbe=\\par <i>Default Behavior:</i>^^'
+    case_sense_names false
+    extension_mapping 'inc=C++', 'no_extension=C++'
+    extract_all true
+    extract_static true
+    file_patterns '*'
+    full_path_names true
+    generate_html true
+    generate_latex false
+    generate_treeview true
+    html_extra_stylesheet 'theme.css'
+    html_timestamp true
+    javadoc_autobrief true
+    project_name 'PhotonVision C++'
+    project_logo '../wpiutil/src/main/native/resources/wpilib-128.png'
+    project_number pubVersion
+    quiet true
+    recursive true
+    strip_code_comments false
+    strip_from_inc_path cppIncludeRoots as String[]
+    strip_from_path cppIncludeRoots as String[]
+    use_mathjax true
+    warnings false
+    warn_if_incomplete_doc true
+    warn_if_undocumented false
+    warn_no_paramdoc true
+
+    //enable doxygen preprocessor expansion of WPI_DEPRECATED to fix MotorController docs
+    enable_preprocessing true
+    macro_expansion true
+    expand_only_predef true
+    predefined "WPI_DEPRECATED(x)=[[deprecated(x)]]\"\\\n" +
+            "\"__cplusplus\"\\\n" +
+            "\"HAL_ENUM(name)=enum name : int32_t"
+
+    if (project.hasProperty('docWarningsAsErrors')) {
+        warn_as_error 'FAIL_ON_WARNINGS'
+    }
+}
+
+tasks.register("zipCppDocs", Zip) {
+    archiveBaseName = zipBaseNameCpp
+    destinationDirectory = outputsFolder
+    dependsOn doxygen
+    from ("$buildDir/docs/doxygen/html")
+    into '/'
+}
+
+// Java
+configurations {
+    javaSource {
+        transitive false
+    }
+}
+
+ext {
+    sharedCvConfigs = [:]
+    staticCvConfigs = [:]
+    useJava = true
+    useCpp = false
+    skipDev = true
+    useDocumentation = true
+}
+
+task generateJavaDocs(type: Javadoc) {
+    def exportedProjects = [
+        ':photon-core',
+        ':photon-server',
+        ':photon-targeting',
+        ':photon-lib'
+    ]
+
+    source exportedProjects.collect { project(it).sourceSets.main.allJava }
+    classpath = files(exportedProjects.collect { project(it).sourceSets.main.compileClasspath })
+    dependsOn project(':photon-core').writeCurrentVersion
+
+    options.links("https://docs.oracle.com/en/java/javase/17/docs/api/")
+    options.addStringOption("tag", "pre:a:Pre-Condition")
+    options.addBooleanOption("Xdoclint:html,missing,reference,syntax", true)
+    options.addBooleanOption('html5', true)
+    failOnError = true
+
+    title = "PhotonVision $pubVersion"
+    ext.entryPoint = "$destinationDir/index.html"
+
+    if (JavaVersion.current().isJava8Compatible() && project.hasProperty('docWarningsAsErrors')) {
+        // Treat javadoc warnings as errors.
+        //
+        // The second argument '-quiet' is a hack. The one parameter
+        // addStringOption() doesn't work, so we add '-quiet', which is added
+        // anyway by gradle. See https://github.com/gradle/gradle/issues/2354.
+        //
+        // See JDK-8200363 (https://bugs.openjdk.java.net/browse/JDK-8200363)
+        // for information about the nonstandard -Xwerror option. JDK 15+ has
+        // -Werror.
+        options.addStringOption('Xwerror', '-quiet')
+    }
+
+    if (JavaVersion.current().isJava11Compatible()) {
+        if (!JavaVersion.current().isJava12Compatible()) {
+            options.addBooleanOption('-no-module-directories', true)
+        }
+        doLast {
+            // This is a work-around for https://bugs.openjdk.java.net/browse/JDK-8211194. Can be removed once that issue is fixed on JDK's side
+            // Since JDK 11, package-list is missing from javadoc output files and superseded by element-list file, but a lot of external tools still need it
+            // Here we generate this file manually
+            new File(destinationDir, 'package-list').text = new File(destinationDir, 'element-list').text
+        }
+    }
+}
+
+tasks.register("zipJavaDocs", Zip) {
+    archiveBaseName = zipBaseNameJava
+    destinationDirectory = outputsFolder
+    dependsOn generateJavaDocs
+    from ("$buildDir/docs/javadoc")
+    into '/'
+}
+
+tasks.register("zipDocs") {
+    dependsOn zipCppDocs
+    dependsOn zipJavaDocs
+}
+
+apply plugin: 'maven-publish'
+
+publishing {
+    publications {
+        java(MavenPublication) {
+            artifact zipJavaDocs
+
+            artifactId = "${baseArtifactIdJava}"
+            groupId artifactGroupIdJava
+            version pubVersion
+        }
+        cpp(MavenPublication) {
+            artifact zipCppDocs
+
+            artifactId = "${baseArtifactIdCpp}"
+            groupId artifactGroupIdCpp
+            version pubVersion
+        }
+    }
+}

photon-client/index.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />

photon-client/package.json

@@ -7,41 +7,42 @@
     "build": "run-p build-only",
     "preview": "vite preview --port 4173",
     "build-only": "vite build",
+    "build-demo": "vite build --mode demo",
     "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
     "format": "prettier --write src/",
     "lint-ci": "eslint . --max-warnings 0 --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
     "format-ci": "prettier --check src/"
   },
   "dependencies": {
-    "@fontsource/prompt": "^5.0.5",
+    "@fontsource/prompt": "^5.0.9",
-    "@mdi/font": "^7.2.96",
+    "@mdi/font": "^7.4.47",
     "@msgpack/msgpack": "^3.0.0-beta2",
-    "axios": "^1.4.0",
+    "axios": "^1.6.3",
     "jspdf": "^2.5.1",
     "pinia": "^2.1.4",
-    "three": "^0.154.0",
+    "three": "^0.160.0",
     "vue": "^2.7.14",
     "vue-router": "^3.6.5",
-    "vuetify": "^2.6.15"
+    "vuetify": "^2.7.1"
   },
   "devDependencies": {
     "@rushstack/eslint-patch": "^1.3.2",
-    "@vue/eslint-config-prettier": "^8.0.0",
-    "@vue/eslint-config-typescript": "^11.0.3",
-    "prettier": "^3.0.0",
     "@types/node": "^16.11.45",
-    "@types/three": "^0.154.0",
+    "@types/three": "^0.160.0",
-    "@vitejs/plugin-vue2": "^2.2.0",
+    "@vitejs/plugin-vue2": "^2.3.1",
-    "@vue/tsconfig": "^0.1.3",
+    "@vue/eslint-config-prettier": "^9.0.0",
+    "@vue/eslint-config-typescript": "^12.0.0",
+    "@vue/tsconfig": "^0.5.1",
     "deepmerge": "^4.3.1",
-    "eslint": "^8.45.0",
+    "eslint": "^8.56.0",
-    "eslint-plugin-vue": "^9.0.0",
+    "eslint-plugin-vue": "^9.19.2",
     "npm-run-all": "^4.1.5",
+    "prettier": "3.2.2",
     "sass": "~1.32",
     "sass-loader": "^13.3.2",
     "terser": "^5.14.2",
-    "typescript": "~4.7.4",
+    "typescript": "^5.3.3",
-    "unplugin-vue-components": "^0.25.1",
+    "unplugin-vue-components": "^0.26.0",
-    "vite": "^4.3.9"
+    "vite": "^4.5.1"
   }
 }

photon-client/src/App.vue

@@ -8,43 +8,45 @@ import PhotonSidebar from "@/components/app/photon-sidebar.vue";
 import PhotonLogView from "@/components/app/photon-log-view.vue";
 import PhotonErrorSnackbar from "@/components/app/photon-error-snackbar.vue";
 
-const websocket = new AutoReconnectingWebsocket(
+const is_demo = import.meta.env.MODE === "demo";
-  `ws://${inject("backendHost")}/websocket_data`,
+if (!is_demo) {
-  () => {
+  const websocket = new AutoReconnectingWebsocket(
-    useStateStore().$patch({ backendConnected: true });
+    `ws://${inject("backendHost")}/websocket_data`,
-  },
+    () => {
-  (data) => {
+      useStateStore().$patch({ backendConnected: true });
-    if (data.log !== undefined) {
+    },
-      useStateStore().addLogFromWebsocket(data.log);
+    (data) => {
+      if (data.log !== undefined) {
+        useStateStore().addLogFromWebsocket(data.log);
+      }
+      if (data.settings !== undefined) {
+        useSettingsStore().updateGeneralSettingsFromWebsocket(data.settings);
+      }
+      if (data.cameraSettings !== undefined) {
+        useCameraSettingsStore().updateCameraSettingsFromWebsocket(data.cameraSettings);
+      }
+      if (data.ntConnectionInfo !== undefined) {
+        useStateStore().updateNTConnectionStatusFromWebsocket(data.ntConnectionInfo);
+      }
+      if (data.metrics !== undefined) {
+        useSettingsStore().updateMetricsFromWebsocket(data.metrics);
+      }
+      if (data.updatePipelineResult !== undefined) {
+        useStateStore().updateBackendResultsFromWebsocket(data.updatePipelineResult);
+      }
+      if (data.mutatePipelineSettings !== undefined && data.cameraIndex !== undefined) {
+        useCameraSettingsStore().changePipelineSettingsInStore(data.mutatePipelineSettings, data.cameraIndex);
+      }
+      if (data.calibrationData !== undefined) {
+        useStateStore().updateCalibrationStateValuesFromWebsocket(data.calibrationData);
+      }
+    },
+    () => {
+      useStateStore().$patch({ backendConnected: false });
     }
-    if (data.settings !== undefined) {
+  );
-      useSettingsStore().updateGeneralSettingsFromWebsocket(data.settings);
+  useStateStore().$patch({ websocket: websocket });
-    }
+}
-    if (data.cameraSettings !== undefined) {
-      useCameraSettingsStore().updateCameraSettingsFromWebsocket(data.cameraSettings);
-    }
-    if (data.ntConnectionInfo !== undefined) {
-      useStateStore().updateNTConnectionStatusFromWebsocket(data.ntConnectionInfo);
-    }
-    if (data.metrics !== undefined) {
-      useSettingsStore().updateMetricsFromWebsocket(data.metrics);
-    }
-    if (data.updatePipelineResult !== undefined) {
-      useStateStore().updateBackendResultsFromWebsocket(data.updatePipelineResult);
-    }
-    if (data.mutatePipelineSettings !== undefined && data.cameraIndex !== undefined) {
-      useCameraSettingsStore().changePipelineSettingsInStore(data.mutatePipelineSettings, data.cameraIndex);
-    }
-    if (data.calibrationData !== undefined) {
-      useStateStore().updateCalibrationStateValuesFromWebsocket(data.calibrationData);
-    }
-  },
-  () => {
-    useStateStore().$patch({ backendConnected: false });
-  }
-);
-
-useStateStore().$patch({ websocket: websocket });
 </script>
 
 <template>

photon-client/src/assets/styles/variables.scss

@@ -7,3 +7,18 @@ $heading-font-family: $default-font;
 .v-application {
   font-family: $default-font !important;
 }
+
+.v-row-group__header {
+  background: #005281 !important;
+}
+.theme--dark.v-data-table
+  > .v-data-table__wrapper
+  > table
+  > tbody
+  > tr:hover:not(.v-data-table__expanded__content):not(.v-data-table__empty-wrapper) {
+  background: #005281 !important;
+}
+
+.v-card__title {
+  word-break: break-word !important;
+}

photon-client/src/components/app/photon-camera-stream.vue

@@ -8,7 +8,7 @@ import PvIcon from "@/components/common/pv-icon.vue";
 
 const props = defineProps<{
   streamType: "Raw" | "Processed";
-  id?: string;
+  id: string;
 }>();
 
 const streamSrc = computed<string>(() => {
@@ -25,8 +25,6 @@ const streamDesc = computed<string>(() => `${props.streamType} Stream View`);
 const streamStyle = computed<StyleValue>(() => {
   if (useStateStore().colorPickingMode) {
     return { width: "100%", cursor: "crosshair" };
-  } else if (streamSrc.value !== loadingImage) {
-    return { width: "100%", cursor: "pointer" };
   }
 
   return { width: "100%" };
@@ -40,11 +38,6 @@ const overlayStyle = computed<StyleValue>(() => {
   }
 });
 
-const handleStreamClick = () => {
-  if (!useStateStore().colorPickingMode && streamSrc.value !== loadingImage) {
-    window.open(streamSrc.value);
-  }
-};
 const handleCaptureClick = () => {
   if (props.streamType === "Raw") {
     useCameraSettingsStore().saveInputSnapshot();
@@ -52,18 +45,19 @@ const handleCaptureClick = () => {
     useCameraSettingsStore().saveOutputSnapshot();
   }
 };
+const handlePopoutClick = () => {
+  window.open(streamSrc.value);
+};
+const handleFullscreenRequest = () => {
+  const stream = document.getElementById(props.id);
+  if (!stream) return;
+  stream.requestFullscreen();
+};
 </script>
 
 <template>
   <div class="stream-container">
-    <img
+    <img :id="id" crossorigin="anonymous" :src="streamSrc" :alt="streamDesc" :style="streamStyle" />
-      :id="id"
-      crossorigin="anonymous"
-      :src="streamSrc"
-      :alt="streamDesc"
-      :style="streamStyle"
-      @click="handleStreamClick"
-    />
     <div class="stream-overlay" :style="overlayStyle">
       <pv-icon
         icon-name="mdi-camera-image"
@@ -71,6 +65,18 @@ const handleCaptureClick = () => {
         class="ma-1 mr-2"
         @click="handleCaptureClick"
       />
+      <pv-icon
+        icon-name="mdi-fullscreen"
+        tooltip="Open this stream in fullscreen"
+        class="ma-1 mr-2"
+        @click="handleFullscreenRequest"
+      />
+      <pv-icon
+        icon-name="mdi-open-in-new"
+        tooltip="Open this stream in a new window"
+        class="ma-1 mr-2"
+        @click="handlePopoutClick"
+      />
     </div>
   </div>
 </template>
@@ -81,6 +87,7 @@ const handleCaptureClick = () => {
 }
 
 .stream-overlay {
+  display: flex;
   opacity: 0;
   transition: 0.1s ease;
   position: absolute;

photon-client/src/components/app/photon-log-view.vue

@@ -47,25 +47,33 @@ document.addEventListener("keydown", (e) => {
 <template>
   <v-dialog v-model="useStateStore().showLogModal" width="1500" dark>
     <v-card dark class="pt-3" color="primary" flat>
-      <v-card-title>
+      <v-row class="heading-container pl-6 pr-6">
-        View Program Logs
+        <v-col>
-        <v-btn color="secondary" style="margin-left: auto" depressed @click="handleLogExport">
+          <v-card-title>View Program Logs</v-card-title>
-          <v-icon left> mdi-download </v-icon>
+        </v-col>
-          Download Current Log
+        <v-col class="align-self-center">
-
+          <v-btn
-          <!-- Special hidden link that gets 'clicked' when the user exports journalctl logs -->
+            color="secondary"
-          <a
+            style="margin-left: auto; max-width: 500px; width: 100%"
-            ref="exportLogFile"
+            depressed
-            style="color: black; text-decoration: none; display: none"
+            @click="handleLogExport"
-            :href="`http://${backendHost}/api/utils/photonvision-journalctl.txt`"
+          >
-            download="photonvision-journalctl.txt"
+            <v-icon left class="open-icon"> mdi-download </v-icon>
-            target="_blank"
+            <span class="open-label">Download Current Log</span>
-          />
-        </v-btn>
-      </v-card-title>
 
+            <!-- Special hidden link that gets 'clicked' when the user exports journalctl logs -->
+            <a
+              ref="exportLogFile"
+              style="color: black; text-decoration: none; display: none"
+              :href="`http://${backendHost}/api/utils/photonvision-journalctl.txt`"
+              download="photonvision-journalctl.txt"
+              target="_blank"
+            />
+          </v-btn>
+        </v-col>
+      </v-row>
       <div class="pr-6 pl-6">
-        <v-btn-toggle v-model="selectedLogLevels" dark multiple class="fill mb-4">
+        <v-btn-toggle v-model="selectedLogLevels" dark multiple class="fill mb-4 overflow-x-auto">
           <v-btn v-for="level in [0, 1, 2, 3]" :key="level" color="secondary" class="fill">
             {{ getLogLevelFromIndex(level) }}
           </v-btn>
@@ -102,4 +110,18 @@ document.addEventListener("keydown", (e) => {
   width: 25%;
   height: 100%;
 }
+@media only screen and (max-width: 512px) {
+  .heading-container {
+    flex-direction: column;
+    padding-bottom: 14px;
+  }
+}
+@media only screen and (max-width: 312px) {
+  .open-icon {
+    margin: 0 !important;
+  }
+  .open-label {
+    display: none;
+  }
+}
 </style>

photon-client/src/components/cameras/CameraCalibrationCard.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { computed, ref } from "vue";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { CalibrationBoardTypes, type Resolution, type VideoFormat } from "@/types/SettingTypes";
+import { CalibrationBoardTypes, type VideoFormat } from "@/types/SettingTypes";
 import JsPDF from "jspdf";
 import { font as PromptRegular } from "@/assets/fonts/PromptRegular";
 import MonoLogo from "@/assets/images/logoMono.png";
@@ -11,38 +11,42 @@ import PvSwitch from "@/components/common/pv-switch.vue";
 import PvSelect from "@/components/common/pv-select.vue";
 import PvNumberInput from "@/components/common/pv-number-input.vue";
 import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
+import { getResolutionString, resolutionsAreEqual } from "@/lib/PhotonUtils";
+import CameraCalibrationInfoCard from "@/components/cameras/CameraCalibrationInfoCard.vue";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 
 const settingsValid = ref(true);
 
-const getCalibrationCoeffs = (resolution: Resolution) => {
+const getUniqueVideoFormatsByResolution = (): VideoFormat[] => {
-  return useCameraSettingsStore().currentCameraSettings.completeCalibrations.find(
-    (cal) => cal.resolution.width === resolution.width && cal.resolution.height === resolution.height
-  );
-};
-const getUniqueVideoResolutions = (): VideoFormat[] => {
   const uniqueResolutions: VideoFormat[] = [];
   useCameraSettingsStore().currentCameraSettings.validVideoFormats.forEach((format, index) => {
-    if (
+    if (!uniqueResolutions.some((v) => resolutionsAreEqual(v.resolution, format.resolution))) {
-      !uniqueResolutions.some(
-        (v) => v.resolution.width === format.resolution.width && v.resolution.height === format.resolution.height
-      )
-    ) {
       format.index = index;
 
-      const calib = getCalibrationCoeffs(format.resolution);
+      const calib = useCameraSettingsStore().getCalibrationCoeffs(format.resolution);
       if (calib !== undefined) {
-        format.standardDeviation = calib.standardDeviation;
+        // Is this the right formula for RMS error? who knows! not me!
-        format.mean = calib.perViewErrors.reduce((a, b) => a + b) / calib.perViewErrors.length;
+        const perViewSumSquareReprojectionError = calib.observations.flatMap((it) =>
-        format.horizontalFOV = 2 * Math.atan2(format.resolution.width / 2, calib.intrinsics[0]) * (180 / Math.PI);
+          it.reprojectionErrors.flatMap((it2) => [it2.x, it2.y])
-        format.verticalFOV = 2 * Math.atan2(format.resolution.height / 2, calib.intrinsics[4]) * (180 / Math.PI);
+        );
+        // For each error, square it, sum the squares, and divide by total points N
+        format.mean = Math.sqrt(
+          perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
+            perViewSumSquareReprojectionError.length
+        );
+
+        format.horizontalFOV =
+          2 * Math.atan2(format.resolution.width / 2, calib.cameraIntrinsics.data[0]) * (180 / Math.PI);
+        format.verticalFOV =
+          2 * Math.atan2(format.resolution.height / 2, calib.cameraIntrinsics.data[4]) * (180 / Math.PI);
         format.diagonalFOV =
           2 *
           Math.atan2(
             Math.sqrt(
               format.resolution.width ** 2 +
-                (format.resolution.height / (calib.intrinsics[4] / calib.intrinsics[0])) ** 2
+                (format.resolution.height / (calib.cameraIntrinsics.data[4] / calib.cameraIntrinsics.data[0])) ** 2
             ) / 2,
-            calib.intrinsics[0]
+            calib.cameraIntrinsics.data[0]
           ) *
           (180 / Math.PI);
       }
@@ -54,9 +58,9 @@ const getUniqueVideoResolutions = (): VideoFormat[] => {
   );
   return uniqueResolutions;
 };
-const getUniqueVideoResolutionStrings = () =>
+const getUniqueVideoResolutionStrings = (): { name: string; value: number }[] =>
-  getUniqueVideoResolutions().map<{ name: string; value: number }>((f) => ({
+  getUniqueVideoFormatsByResolution().map<{ name: string; value: number }>((f) => ({
-    name: `${f.resolution.width} X ${f.resolution.height}`,
+    name: `${getResolutionString(f.resolution)}`,
     // Index won't ever be undefined
     value: f.index || 0
   }));
@@ -71,6 +75,15 @@ const squareSizeIn = ref(1);
 const patternWidth = ref(8);
 const patternHeight = ref(8);
 const boardType = ref<CalibrationBoardTypes>(CalibrationBoardTypes.Chessboard);
+const useMrCalRef = ref(true);
+const useMrCal = computed<boolean>({
+  get() {
+    return useMrCalRef.value && useSettingsStore().general.mrCalWorking;
+  },
+  set(value) {
+    useMrCalRef.value = value && useSettingsStore().general.mrCalWorking;
+  }
+});
 
 const downloadCalibBoard = () => {
   const doc = new JsPDF({ unit: "in", format: "letter" });
@@ -150,9 +163,9 @@ const importCalibrationFromCalibDB = ref();
 const openCalibUploadPrompt = () => {
   importCalibrationFromCalibDB.value.click();
 };
-const readImportedCalibration = (payload: Event) => {
+const readImportedCalibrationFromCalibDB = () => {
-  if (payload.target == null || !payload.target?.files) return;
+  const files = importCalibrationFromCalibDB.value.files;
-  const files: FileList = payload.target.files as FileList;
+  if (files.length === 0) return;
 
   files[0].text().then((text) => {
     useCameraSettingsStore()
@@ -185,7 +198,8 @@ const startCalibration = () => {
     squareSizeIn: squareSizeIn.value,
     patternHeight: patternHeight.value,
     patternWidth: patternWidth.value,
-    boardType: boardType.value
+    boardType: boardType.value,
+    useMrCal: useMrCal.value
   });
   // The Start PnP method already handles updating the backend so only a store update is required
   useCameraSettingsStore().currentCameraSettings.currentPipelineIndex = WebsocketPipelineType.Calib3d;
@@ -214,107 +228,132 @@ const endCalibration = () => {
       isCalibrating.value = false;
     });
 };
+
+let showCalDialog = ref(false);
+let selectedVideoFormat = ref<VideoFormat | undefined>(undefined);
+const setSelectedVideoFormat = (format: VideoFormat) => {
+  selectedVideoFormat.value = format;
+  showCalDialog.value = true;
+};
 </script>
 
 <template>
   <div>
-    <v-card class="pr-6 pb-3" color="primary" dark>
+    <v-card class="mb-3 pr-6 pb-3" color="primary" dark>
       <v-card-title>Camera Calibration</v-card-title>
       <div class="ml-5">
-        <v-row>
+        <v-row v-show="!isCalibrating" class="pb-12">
-          <v-col cols="12" md="6">
+          <v-card-subtitle class="pb-0 mb-0 pl-3">Complete Calibrations</v-card-subtitle>
-            <v-form ref="form" v-model="settingsValid">
+          <v-simple-table fixed-header height="100%" dense class="mt-2">
-              <pv-select
+            <thead>
-                v-model="useStateStore().calibrationData.videoFormatIndex"
+              <tr>
-                label="Resolution"
+                <th>Resolution</th>
-                :select-cols="7"
+                <th>Mean Error</th>
-                :disabled="isCalibrating"
+                <th>Horizontal FOV</th>
-                tooltip="Resolution to calibrate at (you will have to calibrate every resolution you use 3D mode on)"
+                <th>Vertical FOV</th>
-                :items="getUniqueVideoResolutionStrings()"
+                <th>Diagonal FOV</th>
-              />
+              </tr>
-              <pv-select
+            </thead>
-                v-show="isCalibrating"
+            <tbody>
-                v-model="useCameraSettingsStore().currentPipelineSettings.streamingFrameDivisor"
+              <tr
-                label="Decimation"
+                v-for="(value, index) in getUniqueVideoFormatsByResolution()"
-                tooltip="Resolution to which camera frames are downscaled for detection. Calibration still uses full-res"
+                :key="index"
-                :items="calibrationDivisors"
+                title="Click to get calibration specific information"
-                :select-cols="7"
+                @click="setSelectedVideoFormat(value)"
-                @input="
-                  (v) => useCameraSettingsStore().changeCurrentPipelineSetting({ streamingFrameDivisor: v }, false)
-                "
-              />
-              <pv-select
-                v-model="boardType"
-                label="Board Type"
-                tooltip="Calibration board pattern to use"
-                :select-cols="7"
-                :items="['Chessboard', 'Dotboard']"
-                :disabled="isCalibrating"
-              />
-              <pv-number-input
-                v-model="squareSizeIn"
-                label="Pattern Spacing (in)"
-                tooltip="Spacing between pattern features in inches"
-                :disabled="isCalibrating"
-                :rules="[(v) => v > 0 || 'Size must be positive']"
-                :label-cols="5"
-              />
-              <pv-number-input
-                v-model="patternWidth"
-                label="Board Width (in)"
-                tooltip="Width of the board in dots or chessboard squares"
-                :disabled="isCalibrating"
-                :rules="[(v) => v >= 4 || 'Width must be at least 4']"
-                :label-cols="5"
-              />
-              <pv-number-input
-                v-model="patternHeight"
-                label="Board Height (in)"
-                tooltip="Height of the board in dots or chessboard squares"
-                :disabled="isCalibrating"
-                :rules="[(v) => v >= 4 || 'Height must be at least 4']"
-                :label-cols="5"
-              />
-            </v-form>
-          </v-col>
-          <v-col cols="12" md="6">
-            <v-row align="start" class="pb-4 pt-2">
-              <v-simple-table fixed-header height="100%" dense>
-                <thead>
-                  <tr>
-                    <th>Resolution</th>
-                    <th>Mean Error</th>
-                    <th>Standard Deviation</th>
-                    <th>Horizontal FOV</th>
-                    <th>Vertical FOV</th>
-                    <th>Diagonal FOV</th>
-                  </tr>
-                </thead>
-                <tbody>
-                  <tr v-for="(value, index) in getUniqueVideoResolutions()" :key="index">
-                    <td>{{ value.resolution.width }} X {{ value.resolution.height }}</td>
-                    <td>{{ value.mean !== undefined ? value.mean.toFixed(2) + "px" : "-" }}</td>
-                    <td>
-                      {{ value.standardDeviation !== undefined ? value.standardDeviation.toFixed(2) + "px" : "-" }}
-                    </td>
-                    <td>{{ value.horizontalFOV !== undefined ? value.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
-                    <td>{{ value.verticalFOV !== undefined ? value.verticalFOV.toFixed(2) + "°" : "-" }}</td>
-                    <td>{{ value.diagonalFOV !== undefined ? value.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
-                  </tr>
-                </tbody>
-              </v-simple-table>
-            </v-row>
-            <v-row justify="center">
-              <v-chip
-                v-show="isCalibrating"
-                label
-                :color="useStateStore().calibrationData.hasEnoughImages ? 'secondary' : 'gray'"
               >
-                Snapshots: {{ useStateStore().calibrationData.imageCount }} of at least
+                <td>{{ getResolutionString(value.resolution) }}</td>
-                {{ useStateStore().calibrationData.minimumImageCount }}
+                <td>
-              </v-chip>
+                  {{ value.mean !== undefined ? (isNaN(value.mean) ? "NaN" : value.mean.toFixed(2) + "px") : "-" }}
-            </v-row>
+                </td>
-          </v-col>
+                <td>{{ value.horizontalFOV !== undefined ? value.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
+                <td>{{ value.verticalFOV !== undefined ? value.verticalFOV.toFixed(2) + "°" : "-" }}</td>
+                <td>{{ value.diagonalFOV !== undefined ? value.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
+              </tr>
+            </tbody>
+          </v-simple-table>
+        </v-row>
+        <v-divider />
+        <v-row style="display: flex; flex-direction: column" class="mt-4">
+          <v-card-subtitle v-show="!isCalibrating" class="pl-3 pa-0 ma-0"> Configure New Calibration</v-card-subtitle>
+          <v-form ref="form" v-model="settingsValid" class="pl-4 mb-10 pr-5">
+            <pv-select
+              v-model="useStateStore().calibrationData.videoFormatIndex"
+              label="Resolution"
+              :select-cols="7"
+              :disabled="isCalibrating"
+              tooltip="Resolution to calibrate at (you will have to calibrate every resolution you use 3D mode on)"
+              :items="getUniqueVideoResolutionStrings()"
+            />
+            <pv-select
+              v-show="isCalibrating"
+              v-model="useCameraSettingsStore().currentPipelineSettings.streamingFrameDivisor"
+              label="Decimation"
+              tooltip="Resolution to which camera frames are downscaled for detection. Calibration still uses full-res"
+              :items="calibrationDivisors"
+              :select-cols="7"
+              @input="(v) => useCameraSettingsStore().changeCurrentPipelineSetting({ streamingFrameDivisor: v }, false)"
+            />
+            <pv-select
+              v-model="boardType"
+              label="Board Type"
+              tooltip="Calibration board pattern to use"
+              :select-cols="7"
+              :items="['Chessboard', 'Dotboard']"
+              :disabled="isCalibrating"
+            />
+            <pv-number-input
+              v-model="squareSizeIn"
+              label="Pattern Spacing (in)"
+              tooltip="Spacing between pattern features in inches"
+              :disabled="isCalibrating"
+              :rules="[(v) => v > 0 || 'Size must be positive']"
+              :label-cols="5"
+            />
+            <pv-number-input
+              v-model="patternWidth"
+              label="Board Width (squares)"
+              tooltip="Width of the board in dots or chessboard squares"
+              :disabled="isCalibrating"
+              :rules="[(v) => v >= 4 || 'Width must be at least 4']"
+              :label-cols="5"
+            />
+            <pv-number-input
+              v-model="patternHeight"
+              label="Board Height (squares)"
+              tooltip="Height of the board in dots or chessboard squares"
+              :disabled="isCalibrating"
+              :rules="[(v) => v >= 4 || 'Height must be at least 4']"
+              :label-cols="5"
+            />
+            <pv-switch
+              v-model="useMrCal"
+              label="Try using MrCal over OpenCV"
+              :disabled="!useSettingsStore().general.mrCalWorking || isCalibrating"
+              tooltip="If enabled, Photon will (try to) use MrCal instead of OpenCV for camera calibration."
+              :label-cols="5"
+            />
+            <v-banner
+              v-show="!useSettingsStore().general.mrCalWorking"
+              rounded
+              color="red"
+              text-color="white"
+              class="mt-3"
+              icon="mdi-alert-circle-outline"
+            >
+              MrCal JNI could not be loaded! Consult journalctl logs for additional details.
+            </v-banner>
+          </v-form>
+          <v-row justify="center">
+            <v-chip
+              v-show="isCalibrating"
+              label
+              :color="useStateStore().calibrationData.hasEnoughImages ? 'secondary' : 'gray'"
+              class="mb-6"
+            >
+              Snapshots: {{ useStateStore().calibrationData.imageCount }} of at least
+              {{ useStateStore().calibrationData.minimumImageCount }}
+            </v-chip>
+          </v-row>
         </v-row>
         <v-row v-if="isCalibrating">
           <v-col cols="12" class="pt-0">
@@ -387,7 +426,8 @@ const endCalibration = () => {
               :disabled="!settingsValid"
               @click="isCalibrating ? useCameraSettingsStore().takeCalibrationSnapshot() : startCalibration()"
             >
-              {{ isCalibrating ? "Take Snapshot" : "Start Calibration" }}
+              <v-icon left class="calib-btn-icon"> {{ isCalibrating ? "mdi-camera" : "mdi-flag-outline" }} </v-icon>
+              <span class="calib-btn-label">{{ isCalibrating ? "Take Snapshot" : "Start Calibration" }}</span>
             </v-btn>
           </v-col>
           <v-col :cols="6">
@@ -399,7 +439,12 @@ const endCalibration = () => {
               :disabled="!isCalibrating || !settingsValid"
               @click="endCalibration"
             >
-              {{ useStateStore().calibrationData.hasEnoughImages ? "Finish Calibration" : "Cancel Calibration" }}
+              <v-icon left class="calib-btn-icon">
+                {{ useStateStore().calibrationData.hasEnoughImages ? "mdi-flag-checkered" : "mdi-flag-off-outline" }}
+              </v-icon>
+              <span class="calib-btn-label">{{
+                useStateStore().calibrationData.hasEnoughImages ? "Finish Calibration" : "Cancel Calibration"
+              }}</span>
             </v-btn>
           </v-col>
         </v-row>
@@ -413,21 +458,21 @@ const endCalibration = () => {
               :disabled="!settingsValid"
               @click="downloadCalibBoard"
             >
-              <v-icon left> mdi-download </v-icon>
+              <v-icon left class="calib-btn-icon"> mdi-download </v-icon>
-              Generate Board
+              <span class="calib-btn-label">Generate Board</span>
             </v-btn>
           </v-col>
           <v-col :cols="6">
             <v-btn color="secondary" :disabled="isCalibrating" small style="width: 100%" @click="openCalibUploadPrompt">
-              <v-icon left> mdi-upload </v-icon>
+              <v-icon left class="calib-btn-icon"> mdi-upload </v-icon>
-              Import From CalibDB
+              <span class="calib-btn-label">Import From CalibDB</span>
             </v-btn>
             <input
               ref="importCalibrationFromCalibDB"
               type="file"
               accept=".json"
               style="display: none"
-              @change="readImportedCalibration"
+              @change="readImportedCalibrationFromCalibDB"
             />
           </v-col>
         </v-row>
@@ -456,7 +501,7 @@ const endCalibration = () => {
                 {{
                   getUniqueVideoResolutionStrings().find(
                     (v) => v.value === useStateStore().calibrationData.videoFormatIndex
-                  ).name
+                  )?.name
                 }}!
               </v-card-text>
             </template>
@@ -476,12 +521,16 @@ const endCalibration = () => {
         </v-card-actions>
       </v-card>
     </v-dialog>
+    <v-dialog v-model="showCalDialog" width="80em">
+      <CameraCalibrationInfoCard v-if="selectedVideoFormat" :video-format="selectedVideoFormat" />
+    </v-dialog>
   </div>
 </template>
 
 <style scoped lang="scss">
 .v-data-table {
   text-align: center;
+  width: 100%;
 
   th,
   td {
@@ -491,6 +540,7 @@ const endCalibration = () => {
 
   tbody :hover td {
     background-color: #005281 !important;
+    cursor: pointer;
   }
 
   ::-webkit-scrollbar {
@@ -509,4 +559,13 @@ const endCalibration = () => {
     border-radius: 10px;
   }
 }
+
+@media only screen and (max-width: 512px) {
+  .calib-btn-icon {
+    margin: 0 !important;
+  }
+  .calib-btn-label {
+    display: none;
+  }
+}
 </style>

photon-client/src/components/cameras/CameraCalibrationInfoCard.vue

@@ -0,0 +1,306 @@
+<script setup lang="ts">
+import type { BoardObservation, CameraCalibrationResult, VideoFormat } from "@/types/SettingTypes";
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { useStateStore } from "@/stores/StateStore";
+import { ref } from "vue";
+import loadingImage from "@/assets/images/loading.svg";
+import { getResolutionString, parseJsonFile } from "@/lib/PhotonUtils";
+
+const props = defineProps<{
+  videoFormat: VideoFormat;
+}>();
+
+const getMeanFromView = (o: BoardObservation) => {
+  // Is this the right formula for RMS error? who knows! not me!
+  const perViewSumSquareReprojectionError = o.reprojectionErrors.flatMap((it2) => [it2.x, it2.y]);
+
+  // For each error, square it, sum the squares, and divide by total points N
+  return Math.sqrt(
+    perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
+      perViewSumSquareReprojectionError.length
+  );
+};
+
+// Import and export functions
+const downloadCalibration = () => {
+  const calibData = useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution);
+  if (calibData === undefined) {
+    useStateStore().showSnackbarMessage({
+      color: "error",
+      message:
+        "Calibration data isn't available for the requested resolution, please calibrate the requested resolution first"
+    });
+    return;
+  }
+
+  const camUniqueName = useCameraSettingsStore().currentCameraSettings.uniqueName;
+  const filename = `photon_calibration_${camUniqueName}_${calibData.resolution.width}x${calibData.resolution.height}.json`;
+  const fileData = JSON.stringify(calibData);
+
+  const element = document.createElement("a");
+  element.style.display = "none";
+  element.setAttribute("href", "data:text/plain;charset=utf-8," + encodeURIComponent(fileData));
+  element.setAttribute("download", filename);
+
+  document.body.appendChild(element);
+  element.click();
+  document.body.removeChild(element);
+};
+const importCalibrationFromPhotonJson = ref();
+const openUploadPhotonCalibJsonPrompt = () => {
+  importCalibrationFromPhotonJson.value.click();
+};
+const importCalibration = async () => {
+  const files = importCalibrationFromPhotonJson.value.files;
+  if (files.length === 0) return;
+  const uploadedJson = files[0];
+
+  const data = await parseJsonFile<CameraCalibrationResult>(uploadedJson);
+
+  if (
+    data.resolution.height != props.videoFormat.resolution.height ||
+    data.resolution.width != props.videoFormat.resolution.width
+  ) {
+    useStateStore().showSnackbarMessage({
+      color: "error",
+      message: `The resolution of the calibration export doesn't match the current resolution ${props.videoFormat.resolution.height}x${props.videoFormat.resolution.width}`
+    });
+    return;
+  }
+
+  useCameraSettingsStore()
+    .importCalibrationFromData({ calibration: data })
+    .then((response) => {
+      useStateStore().showSnackbarMessage({
+        color: "success",
+        message: response.data.text || response.data
+      });
+    })
+    .catch((error) => {
+      if (error.response) {
+        useStateStore().showSnackbarMessage({
+          color: "error",
+          message: error.response.data.text || error.response.data
+        });
+      } else if (error.request) {
+        useStateStore().showSnackbarMessage({
+          color: "error",
+          message: "Error while trying to process the request! The backend didn't respond."
+        });
+      } else {
+        useStateStore().showSnackbarMessage({
+          color: "error",
+          message: "An error occurred while trying to process the request."
+        });
+      }
+    });
+};
+
+interface ObservationDetails {
+  snapshotSrc: any;
+  mean: number;
+  index: number;
+}
+const getObservationDetails = (): ObservationDetails[] | undefined => {
+  return useCameraSettingsStore()
+    .getCalibrationCoeffs(props.videoFormat.resolution)
+    ?.observations.map((o, i) => ({
+      index: i,
+      mean: parseFloat(getMeanFromView(o).toFixed(2)),
+      snapshotSrc: o.includeObservationInCalibration ? "data:image/png;base64," + o.snapshotData.data : loadingImage
+    }));
+};
+</script>
+
+<template>
+  <v-card color="primary" class="pa-6" dark>
+    <v-row>
+      <v-col cols="12" md="5">
+        <v-card-title class="pl-0 ml-0"
+          ><span class="text-no-wrap" style="white-space: pre !important">Calibration Details: </span
+          ><span class="text-no-wrap"
+            >{{ useCameraSettingsStore().currentCameraName }}@{{ getResolutionString(videoFormat.resolution) }}</span
+          ></v-card-title
+        >
+      </v-col>
+      <v-col>
+        <v-btn color="secondary" class="mt-4" style="width: 100%" @click="openUploadPhotonCalibJsonPrompt">
+          <v-icon left> mdi-import</v-icon>
+          <span>Import</span>
+        </v-btn>
+        <input
+          ref="importCalibrationFromPhotonJson"
+          type="file"
+          accept=".json"
+          style="display: none"
+          @change="importCalibration"
+        />
+      </v-col>
+      <v-col>
+        <v-btn
+          color="secondary"
+          class="mt-4"
+          :disabled="useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution) === undefined"
+          style="width: 100%"
+          @click="downloadCalibration"
+        >
+          <v-icon left>mdi-export</v-icon>
+          <span>Export</span>
+        </v-btn>
+      </v-col>
+    </v-row>
+    <v-row
+      v-if="useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution) !== undefined"
+      class="pt-2"
+    >
+      <v-card-subtitle>Calibration Details</v-card-subtitle>
+      <v-simple-table dense style="width: 100%" class="pl-2 pr-2">
+        <template #default>
+          <thead>
+            <tr>
+              <th class="text-left">Name</th>
+              <th class="text-left">Value</th>
+            </tr>
+          </thead>
+          <tbody>
+            <tr>
+              <td>Fx</td>
+              <td>
+                {{
+                  useCameraSettingsStore()
+                    .getCalibrationCoeffs(props.videoFormat.resolution)
+                    ?.cameraIntrinsics.data[0].toFixed(2) || 0.0
+                }}
+                mm
+              </td>
+            </tr>
+            <tr>
+              <td>Fy</td>
+              <td>
+                {{
+                  useCameraSettingsStore()
+                    .getCalibrationCoeffs(props.videoFormat.resolution)
+                    ?.cameraIntrinsics.data[4].toFixed(2) || 0.0
+                }}
+                mm
+              </td>
+            </tr>
+            <tr>
+              <td>Cx</td>
+              <td>
+                {{
+                  useCameraSettingsStore()
+                    .getCalibrationCoeffs(props.videoFormat.resolution)
+                    ?.cameraIntrinsics.data[2].toFixed(2) || 0.0
+                }}
+                px
+              </td>
+            </tr>
+            <tr>
+              <td>Cy</td>
+              <td>
+                {{
+                  useCameraSettingsStore()
+                    .getCalibrationCoeffs(props.videoFormat.resolution)
+                    ?.cameraIntrinsics.data[5].toFixed(2) || 0.0
+                }}
+                px
+              </td>
+            </tr>
+            <tr>
+              <td>Distortion</td>
+              <td>
+                {{
+                  useCameraSettingsStore()
+                    .getCalibrationCoeffs(props.videoFormat.resolution)
+                    ?.distCoeffs.data.map((it) => parseFloat(it.toFixed(3))) || []
+                }}
+              </td>
+            </tr>
+            <tr>
+              <td>Mean Err</td>
+              <td>
+                {{
+                  videoFormat.mean !== undefined
+                    ? isNaN(videoFormat.mean)
+                      ? "NaN"
+                      : videoFormat.mean.toFixed(2) + "px"
+                    : "-"
+                }}
+              </td>
+            </tr>
+            <tr>
+              <td>Horizontal FOV</td>
+              <td>{{ videoFormat.horizontalFOV !== undefined ? videoFormat.horizontalFOV.toFixed(2) + "°" : "-" }}</td>
+            </tr>
+            <tr>
+              <td>Vertical FOV</td>
+              <td>{{ videoFormat.verticalFOV !== undefined ? videoFormat.verticalFOV.toFixed(2) + "°" : "-" }}</td>
+            </tr>
+            <tr>
+              <td>Diagonal FOV</td>
+              <td>{{ videoFormat.diagonalFOV !== undefined ? videoFormat.diagonalFOV.toFixed(2) + "°" : "-" }}</td>
+            </tr>
+            <!-- Board warp, only shown for mrcal-calibrated cameras -->
+            <tr
+              v-if="
+                useCameraSettingsStore().getCalibrationCoeffs(props.videoFormat.resolution)?.calobjectWarp?.length === 2
+              "
+            >
+              <td>Board warp, X/Y</td>
+              <td>
+                {{
+                  useCameraSettingsStore()
+                    .getCalibrationCoeffs(props.videoFormat.resolution)
+                    ?.calobjectWarp?.map((it) => (it * 1000).toFixed(2) + " mm")
+                    .join(" / ")
+                }}
+              </td>
+            </tr>
+          </tbody>
+        </template>
+      </v-simple-table>
+      <hr style="width: 100%" class="ma-6" />
+      <v-card-subtitle>Per Observation Details</v-card-subtitle>
+      <v-data-table
+        dense
+        style="width: 100%"
+        class="pl-2 pr-2"
+        :headers="[
+          { text: 'Observation Id', value: 'index' },
+          { text: 'Mean Reprojection Error', value: 'mean' }
+        ]"
+        :items="getObservationDetails()"
+        item-key="index"
+        show-expand
+        expand-icon="mdi-eye"
+      >
+        <template #expanded-item="{ headers, item }">
+          <td :colspan="headers.length">
+            <div style="display: flex; justify-content: center; width: 100%">
+              <img :src="item.snapshotSrc" alt="observation image" class="snapshot-preview pt-2 pb-2" />
+            </div>
+          </td>
+        </template>
+      </v-data-table>
+    </v-row>
+    <v-row v-else class="pt-2 mb-0 pb-0">
+      The selected video format doesn't have any additional information as it has yet to be calibrated.
+    </v-row>
+  </v-card>
+</template>
+
+<style scoped>
+.v-data-table {
+  background-color: #006492 !important;
+}
+.snapshot-preview {
+  max-width: 55%;
+}
+
+@media only screen and (max-width: 512px) {
+  .snapshot-preview {
+    max-width: 100%;
+  }
+}
+</style>

photon-client/src/components/cameras/CameraControlCard.vue

@@ -0,0 +1,210 @@
+<script setup lang="ts">
+import { ref } from "vue";
+import axios from "axios";
+import { useStateStore } from "@/stores/StateStore";
+
+interface SnapshotMetadata {
+  snapshotName: string;
+  cameraNickname: string;
+  streamType: "input" | "output";
+  timeCreated: Date;
+}
+const getSnapshotMetadataFromName = (snapshotName: string): SnapshotMetadata => {
+  snapshotName = snapshotName.replace(/\.[^/.]+$/, "");
+
+  const data = snapshotName.split("_");
+
+  const cameraName = data.slice(0, data.length - 2).join("_");
+  const streamType = data[data.length - 2] as "input" | "output";
+  const dateStr = data[data.length - 1];
+
+  const year = parseInt(dateStr.substring(0, 4), 10);
+  const month = parseInt(dateStr.substring(5, 7), 10) - 1; // Months are zero-based
+  const day = parseInt(dateStr.substring(8, 10), 10);
+  const hours = parseInt(dateStr.substring(11, 13), 10);
+  const minutes = parseInt(dateStr.substring(13, 15), 10);
+  const seconds = parseInt(dateStr.substring(15, 17), 10);
+  const milliseconds = parseInt(dateStr.substring(17), 10);
+
+  return {
+    snapshotName: snapshotName,
+    cameraNickname: cameraName,
+    streamType: streamType,
+    timeCreated: new Date(year, month, day, hours, minutes, seconds, milliseconds)
+  };
+};
+
+interface Snapshot {
+  index: number;
+  snapshotName: string;
+  snapshotShortName: string;
+  cameraUniqueName: string;
+  cameraNickname: string;
+  streamType: "input" | "output";
+  timeCreated: Date;
+  snapshotSrc: string;
+}
+const imgData = ref<Snapshot[]>([]);
+const fetchSnapshots = () => {
+  axios
+    .get("/utils/getImageSnapshots")
+    .then((response) => {
+      imgData.value = response.data.map(
+        (snapshotData: { snapshotName: string; cameraUniqueName: string; snapshotData: string }, index) => {
+          const metadata = getSnapshotMetadataFromName(snapshotData.snapshotName);
+
+          return {
+            index: index,
+            snapshotName: snapshotData.snapshotName,
+            snapshotShortName: metadata.snapshotName,
+            cameraUniqueName: snapshotData.cameraUniqueName,
+            cameraNickname: metadata.cameraNickname,
+            streamType: metadata.streamType,
+            timeCreated: metadata.timeCreated,
+            snapshotSrc: "data:image/jpg;base64," + snapshotData.snapshotData
+          };
+        }
+      );
+      showSnapshotViewerDialog.value = true;
+    })
+    .catch((error) => {
+      if (error.response) {
+        useStateStore().showSnackbarMessage({
+          color: "error",
+          message: error.response.data.text || error.response.data
+        });
+      } else if (error.request) {
+        useStateStore().showSnackbarMessage({
+          color: "error",
+          message: "Error while trying to process the request! The backend didn't respond."
+        });
+      } else {
+        useStateStore().showSnackbarMessage({
+          color: "error",
+          message: "An error occurred while trying to process the request."
+        });
+      }
+    });
+};
+const showSnapshotViewerDialog = ref(false);
+const expanded = ref([]);
+</script>
+
+<template>
+  <v-card dark class="pr-6 pb-3" style="background-color: #006492">
+    <v-card-title>Camera Control</v-card-title>
+    <v-row class="pl-6">
+      <v-col>
+        <v-btn color="secondary" @click="fetchSnapshots">
+          <v-icon left class="open-icon"> mdi-folder </v-icon>
+          <span class="open-label">Show Saved Snapshots</span>
+        </v-btn>
+      </v-col>
+    </v-row>
+    <v-dialog v-model="showSnapshotViewerDialog">
+      <v-card dark class="pt-3 pl-5 pr-5" color="primary" flat>
+        <v-card-title> View Saved Frame Snapshots </v-card-title>
+        <v-divider />
+        <v-card-text v-if="imgData.length === 0" style="font-size: 18px; font-weight: 600" class="pt-4">
+          There are no snapshots saved
+        </v-card-text>
+        <div v-else class="pb-2">
+          <v-data-table
+            v-model:expanded="expanded"
+            :headers="[
+              { text: 'Snapshot Name', value: 'snapshotShortName', sortable: false },
+              { text: 'Camera Unique Name', value: 'cameraUniqueName' },
+              { text: 'Camera Nickname', value: 'cameraNickname' },
+              { text: 'Stream Type', value: 'streamType' },
+              { text: 'Time Created', value: 'timeCreated' },
+              { text: 'Actions', value: 'actions', sortable: false }
+            ]"
+            :items="imgData"
+            group-by="cameraUniqueName"
+            class="elevation-0"
+            item-key="index"
+            show-expand
+            expand-icon="mdi-eye"
+          >
+            <template #expanded-item="{ headers, item }">
+              <td :colspan="headers.length">
+                <div style="display: flex; justify-content: center; width: 100%">
+                  <img :src="item.snapshotSrc" alt="snapshot-image" class="snapshot-preview pt-2 pb-2" />
+                </div>
+              </td>
+            </template>
+            <!-- eslint-disable-next-line vue/valid-v-slot-->
+            <template #item.actions="{ item }">
+              <div style="display: flex; justify-content: center">
+                <a :download="item.snapshotName" :href="item.snapshotSrc">
+                  <v-icon small> mdi-download </v-icon>
+                </a>
+              </div>
+            </template>
+          </v-data-table>
+          <span
+            >Snapshot Timestamps may be incorrect as they depend on when the coprocessor was last connected to the
+            internet</span
+          >
+        </div>
+      </v-card>
+    </v-dialog>
+  </v-card>
+</template>
+
+<style scoped lang="scss">
+.v-divider {
+  border-color: white !important;
+}
+.v-btn {
+  width: 100%;
+}
+.v-data-table {
+  text-align: center;
+  background-color: #006492 !important;
+
+  th,
+  td {
+    background-color: #005281 !important;
+    font-size: 1rem !important;
+  }
+
+  tbody :hover tr {
+    background-color: #005281 !important;
+  }
+
+  ::-webkit-scrollbar {
+    width: 0;
+    height: 0.55em;
+    border-radius: 5px;
+  }
+
+  ::-webkit-scrollbar-track {
+    -webkit-box-shadow: inset 0 0 6px rgba(0, 0, 0, 0.3);
+    border-radius: 10px;
+  }
+
+  ::-webkit-scrollbar-thumb {
+    background-color: #ffd843;
+    border-radius: 10px;
+  }
+}
+
+.snapshot-preview {
+  max-width: 55%;
+}
+
+@media only screen and (max-width: 512px) {
+  .snapshot-preview {
+    max-width: 100%;
+  }
+}
+@media only screen and (max-width: 351px) {
+  .open-icon {
+    margin: 0 !important;
+  }
+  .open-label {
+    display: none;
+  }
+}
+</style>

photon-client/src/components/cameras/CameraSettingsCard.vue

@@ -3,22 +3,79 @@ import PvSelect from "@/components/common/pv-select.vue";
 import PvNumberInput from "@/components/common/pv-number-input.vue";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { useStateStore } from "@/stores/StateStore";
-import { ref, watchEffect } from "vue";
+import { computed, ref, watchEffect } from "vue";
+import { type CameraSettingsChangeRequest, ValidQuirks } from "@/types/SettingTypes";
 
-const currentFov = ref();
+const tempSettingsStruct = ref<CameraSettingsChangeRequest>({
+  fov: useCameraSettingsStore().currentCameraSettings.fov.value,
+  quirksToChange: Object.assign({}, useCameraSettingsStore().currentCameraSettings.cameraQuirks.quirks)
+});
+
+const arducamSelectWrapper = computed<number>({
+  get: () => {
+    if (tempSettingsStruct.value.quirksToChange.ArduOV9281) return 1;
+    else if (tempSettingsStruct.value.quirksToChange.ArduOV2311) return 2;
+    else return 0;
+  },
+  set: (v) => {
+    switch (v) {
+      case 1:
+        tempSettingsStruct.value.quirksToChange.ArduOV9281 = true;
+        tempSettingsStruct.value.quirksToChange.ArduOV2311 = false;
+        break;
+      case 2:
+        tempSettingsStruct.value.quirksToChange.ArduOV9281 = false;
+        tempSettingsStruct.value.quirksToChange.ArduOV2311 = true;
+        break;
+      default:
+        tempSettingsStruct.value.quirksToChange.ArduOV9281 = false;
+        tempSettingsStruct.value.quirksToChange.ArduOV2311 = false;
+        break;
+    }
+  }
+});
+
+const currentCameraIsArducam = computed<boolean>(
+  () => useCameraSettingsStore().currentCameraSettings.cameraQuirks.quirks.ArduCamCamera
+);
+
+const settingsHaveChanged = (): boolean => {
+  const a = tempSettingsStruct.value;
+  const b = useCameraSettingsStore().currentCameraSettings;
+
+  for (const q in ValidQuirks) {
+    if (a.quirksToChange[q] != b.cameraQuirks.quirks[q]) return true;
+  }
+
+  return a.fov != b.fov.value;
+};
+
+const resetTempSettingsStruct = () => {
+  tempSettingsStruct.value.fov = useCameraSettingsStore().currentCameraSettings.fov.value;
+  tempSettingsStruct.value.quirksToChange = Object.assign(
+    {},
+    useCameraSettingsStore().currentCameraSettings.cameraQuirks.quirks
+  );
+};
 
 const saveCameraSettings = () => {
   useCameraSettingsStore()
-    .updateCameraSettings({ fov: currentFov.value }, false)
+    .updateCameraSettings(tempSettingsStruct.value)
     .then((response) => {
-      useCameraSettingsStore().currentCameraSettings.fov.value = currentFov.value;
       useStateStore().showSnackbarMessage({
         color: "success",
         message: response.data.text || response.data
       });
+
+      // Update the local settings cause the backend checked their validity. Assign is to deref value
+      useCameraSettingsStore().currentCameraSettings.fov.value = tempSettingsStruct.value.fov;
+      useCameraSettingsStore().currentCameraSettings.cameraQuirks.quirks = Object.assign(
+        {},
+        tempSettingsStruct.value.quirksToChange
+      );
     })
     .catch((error) => {
-      currentFov.value = useCameraSettingsStore().currentCameraSettings.fov.value;
+      resetTempSettingsStruct();
       if (error.response) {
         useStateStore().showSnackbarMessage({
           color: "error",
@@ -39,7 +96,8 @@ const saveCameraSettings = () => {
 };
 
 watchEffect(() => {
-  currentFov.value = useCameraSettingsStore().currentCameraSettings.fov.value;
+  // Reset temp settings on remote camera settings change
+  resetTempSettingsStruct();
 });
 </script>
 
@@ -52,15 +110,9 @@ watchEffect(() => {
         label="Camera"
         :items="useCameraSettingsStore().cameraNames"
         :select-cols="8"
-        @input="
-          (args) => {
-            currentFov = useCameraSettingsStore().cameras[args].fov.value;
-            useCameraSettingsStore().setCurrentCameraIndex(args);
-          }
-        "
       />
       <pv-number-input
-        v-model="currentFov"
+        v-model="tempSettingsStruct.fov"
         :tooltip="
           !useCameraSettingsStore().currentCameraSettings.fov.managedByVendor
             ? 'Field of view (in degrees) of the camera measured across the diagonal of the frame, in a video mode which covers the whole sensor area.'
@@ -70,12 +122,24 @@ watchEffect(() => {
         :disabled="useCameraSettingsStore().currentCameraSettings.fov.managedByVendor"
         :label-cols="4"
       />
+      <pv-select
+        v-show="currentCameraIsArducam"
+        v-model="arducamSelectWrapper"
+        label="Arducam Model"
+        :items="[
+          { name: 'None', value: 0, disabled: true },
+          { name: 'OV9281', value: 1 },
+          { name: 'OV2311', value: 2 }
+        ]"
+        :select-cols="8"
+      />
       <br />
       <v-btn
-        style="margin-top: 10px"
+        class="mt-2 mb-3"
+        style="width: 100%"
         small
         color="secondary"
-        :disabled="currentFov === useCameraSettingsStore().currentCameraSettings.fov.value"
+        :disabled="!settingsHaveChanged()"
         @click="saveCameraSettings"
       >
         <v-icon left> mdi-content-save </v-icon>

photon-client/src/components/cameras/CamerasView.vue

@@ -41,7 +41,12 @@ const fpsTooLow = computed<boolean>(() => {
 </script>
 
 <template>
-  <v-card class="mb-3 pb-3 pa-4" color="primary" dark>
+  <v-card
+    id="camera-settings-camera-view-card"
+    class="camera-settings-camera-view-card mb-3 pb-3 pa-4"
+    color="primary"
+    dark
+  >
     <v-card-title
       class="pb-0 mb-2 pl-4 pt-1"
       style="min-height: 50px; justify-content: space-between; align-content: center"
@@ -78,10 +83,20 @@ const fpsTooLow = computed<boolean>(() => {
     </v-card-title>
     <div class="stream-container pb-4">
       <div class="stream">
-        <photon-camera-stream v-show="value.includes(0)" stream-type="Raw" style="max-width: 100%" />
+        <photon-camera-stream
+          v-show="value.includes(0)"
+          id="input-camera-stream"
+          stream-type="Raw"
+          style="max-width: 100%"
+        />
       </div>
       <div class="stream">
-        <photon-camera-stream v-show="value.includes(1)" stream-type="Processed" style="max-width: 100%" />
+        <photon-camera-stream
+          v-show="value.includes(1)"
+          id="output-camera-stream"
+          stream-type="Processed"
+          style="max-width: 100%"
+        />
       </div>
     </div>
     <v-divider />
@@ -93,16 +108,16 @@ const fpsTooLow = computed<boolean>(() => {
           class="fill"
           :disabled="useCameraSettingsStore().isDriverMode || useCameraSettingsStore().isCalibrationMode"
         >
-          <v-icon>mdi-import</v-icon>
+          <v-icon left class="mode-btn-icon">mdi-import</v-icon>
-          <span>Raw</span>
+          <span class="mode-btn-label">Raw</span>
         </v-btn>
         <v-btn
           color="secondary"
           class="fill"
           :disabled="useCameraSettingsStore().isDriverMode || useCameraSettingsStore().isCalibrationMode"
         >
-          <v-icon>mdi-export</v-icon>
+          <v-icon left class="mode-btn-icon">mdi-export</v-icon>
-          <span>Processed</span>
+          <span class="mode-btn-label">Processed</span>
         </v-btn>
       </v-btn-toggle>
     </div>
@@ -146,4 +161,12 @@ th {
     max-width: 50%;
   }
 }
+@media only screen and (max-width: 351px) {
+  .mode-btn-icon {
+    margin: 0 !important;
+  }
+  .mode-btn-label {
+    display: none;
+  }
+}
 </style>

photon-client/src/components/common/pv-range-slider.vue

@@ -40,12 +40,21 @@ const localValue = computed<[number, number]>({
   }
 });
 
-const changeFromSlot = (v: number, i: number) => {
+const changeFromSlot = (v: string, i: number) => {
+  // v comes in as a string, not a number, for some reason
+  // if v is undefined, take a guess and set it to 0
+  const val = Math.max(props.min, Math.min(parseFloat(v) || 0, props.max));
+
   // localValue.value must be replaced for a reactive change to take place
   const temp = localValue.value;
-  temp[i] = v;
+  temp[i] = val;
   localValue.value = temp;
 };
+
+const checkNumberRange = (v: string): boolean => {
+  const val: number = parseFloat(v);
+  return isFinite(val) && val >= props.min && val <= props.max;
+};
 </script>
 
 <template>
@@ -79,6 +88,7 @@ const changeFromSlot = (v: number, i: number) => {
               :max="max"
               :min="min"
               :step="step"
+              :rules="[checkNumberRange]"
               type="number"
               style="width: 60px"
               @input="(v) => changeFromSlot(v, 0)"
@@ -95,6 +105,7 @@ const changeFromSlot = (v: number, i: number) => {
               :max="max"
               :min="min"
               :step="step"
+              :rules="[checkNumberRange]"
               type="number"
               style="width: 60px"
               @input="(v) => changeFromSlot(v, 1)"

photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

@@ -7,6 +7,7 @@ import { computed, ref } from "vue";
 import PvIcon from "@/components/common/pv-icon.vue";
 import PvInput from "@/components/common/pv-input.vue";
 import { PipelineType } from "@/types/PipelineTypes";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 
 const changeCurrentCameraIndex = (index: number) => {
   useCameraSettingsStore().setCurrentCameraIndex(index, true);
@@ -24,6 +25,9 @@ const changeCurrentCameraIndex = (index: number) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
+      break;
   }
 };
 
@@ -121,6 +125,18 @@ const cancelPipelineNameEdit = () => {
 const showPipelineCreationDialog = ref(false);
 const newPipelineName = ref("");
 const newPipelineType = ref<WebsocketPipelineType>(useCameraSettingsStore().currentWebsocketPipelineType);
+const validNewPipelineTypes = computed(() => {
+  const pipelineTypes = [
+    { name: "Reflective", value: WebsocketPipelineType.Reflective },
+    { name: "Colored Shape", value: WebsocketPipelineType.ColoredShape },
+    { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
+    { name: "Aruco", value: WebsocketPipelineType.Aruco }
+  ];
+  if (useSettingsStore().general.rknnSupported) {
+    pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
+  }
+  return pipelineTypes;
+});
 const showCreatePipelineDialog = () => {
   newPipelineName.value = "";
   newPipelineType.value = useCameraSettingsStore().currentWebsocketPipelineType;
@@ -154,6 +170,9 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
     { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
     { name: "Aruco", value: WebsocketPipelineType.Aruco }
   ];
+  if (useSettingsStore().general.rknnSupported) {
+    pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
+  }
 
   if (useCameraSettingsStore().isDriverMode) {
     pipelineTypes.push({ name: "Driver Mode", value: WebsocketPipelineType.DriverMode });
@@ -208,6 +227,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
+      break;
   }
 });
 </script>
@@ -350,12 +372,7 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
             :select-cols="12 - 3"
             label="Tracking Type"
             tooltip="Pipeline type, which changes the type of processing that will happen on input frames"
-            :items="[
+            :items="validNewPipelineTypes"
-              { name: 'Reflective', value: WebsocketPipelineType.Reflective },
-              { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
-              { name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
-              { name: 'Aruco', value: WebsocketPipelineType.Aruco }
-            ]"
           />
         </v-card-text>
         <v-divider />

photon-client/src/components/dashboard/CamerasCard.vue

@@ -29,61 +29,83 @@ const fpsTooLow = computed<boolean>(() => {
 
   return currFPS - targetFPS < -5 && currFPS !== 0 && !driverMode && gpuAccel && isReflective;
 });
+
+const performanceRecommendation = computed<string>(() => {
+  if (
+    fpsTooLow.value &&
+    !useCameraSettingsStore().currentPipelineSettings.inputShouldShow &&
+    useCameraSettingsStore().currentPipelineSettings.pipelineType === PipelineType.Reflective
+  ) {
+    return "HSV thresholds are too broad; narrow them for better performance";
+  } else if (fpsTooLow.value && useCameraSettingsStore().currentPipelineSettings.inputShouldShow) {
+    return "Stop viewing the raw stream for better performance";
+  } else {
+    return `${Math.min(Math.round(useStateStore().currentPipelineResults?.latency || 0), 9999)} ms latency`;
+  }
+});
 </script>
 
 <template>
   <v-card color="primary" height="100%" style="display: flex; flex-direction: column" dark>
-    <v-card-title
+    <v-row>
-      class="pb-0 mb-0 pl-4 pt-1"
+      <v-col class="align-self-center text-no-wrap">
-      style="min-height: 50px; justify-content: space-between; align-content: center"
+        <v-card-title>Cameras</v-card-title>
-    >
+      </v-col>
-      <div class="pt-2">
+      <v-col class="align-self-center" style="text-align: right; margin-right: 12px; padding-left: 24px">
-        <span class="mr-4">Cameras</span>
         <v-chip
           label
           :color="fpsTooLow ? 'error' : 'transparent'"
           :text-color="fpsTooLow ? '#C7EA46' : '#ff4d00'"
           style="font-size: 1rem; padding: 0; margin: 0"
         >
-          <span class="pr-1">
+          <span class="pr-1"
-            Processing @ {{ Math.round(useStateStore().currentPipelineResults?.fps || 0) }}&nbsp;FPS &ndash;
+            >Processing @ {{ Math.round(useStateStore().currentPipelineResults?.fps || 0) }}&nbsp;FPS &ndash;</span
-          </span>
+          ><span>{{ performanceRecommendation }}</span>
-          <span
-            v-if="
-              fpsTooLow &&
-              !useCameraSettingsStore().currentPipelineSettings.inputShouldShow &&
-              useCameraSettingsStore().currentPipelineSettings.pipelineType === PipelineType.Reflective
-            "
-          >
-            HSV thresholds are too broad; narrow them for better performance
-          </span>
-          <span v-else-if="fpsTooLow && useCameraSettingsStore().currentPipelineSettings.inputShouldShow">
-            stop viewing the raw stream for better performance
-          </span>
-          <span v-else>
-            {{ Math.min(Math.round(useStateStore().currentPipelineResults?.latency || 0), 9999) }} ms latency
-          </span>
         </v-chip>
-      </div>
+      </v-col>
-      <div>
+      <v-col
+        class="align-self-center"
+        style="
+          width: min-content;
+          flex-grow: 0;
+          display: flex;
+          justify-content: flex-end;
+          margin-right: 24px;
+          padding: 0;
+        "
+      >
         <v-switch
           v-model="driverMode"
           :disabled="useCameraSettingsStore().isCalibrationMode || useCameraSettingsStore().pipelineNames.length === 0"
           label="Driver Mode"
-          style="margin-left: auto"
+          style="margin: 0; padding: 0; padding-left: 18px; margin-top: 14px"
           color="accent"
-          class="pt-2"
         />
-      </div>
+      </v-col>
-    </v-card-title>
+    </v-row>
     <v-divider style="border-color: white" />
-    <v-row class="pl-3 pr-3 pt-3 pb-3" style="flex-wrap: nowrap; justify-content: center">
+    <v-row class="stream-viewer-container pa-3">
-      <v-col v-show="value.includes(0)" style="max-width: 500px; display: flex; align-items: center">
+      <v-col v-show="value.includes(0)" class="stream-view">
         <photon-camera-stream id="input-camera-stream" stream-type="Raw" style="width: 100%; height: auto" />
       </v-col>
-      <v-col v-show="value.includes(1)" style="max-width: 500px; display: flex; align-items: center">
+      <v-col v-show="value.includes(1)" class="stream-view">
         <photon-camera-stream id="output-camera-stream" stream-type="Processed" style="width: 100%; height: auto" />
       </v-col>
     </v-row>
   </v-card>
 </template>
+
+<style scoped>
+.stream-viewer-container {
+  display: flex;
+  justify-content: center;
+}
+.stream-view {
+  max-width: 500px;
+}
+@media only screen and (max-width: 512px) {
+  .stream-view {
+    min-width: 80%;
+  }
+}
+</style>

photon-client/src/components/dashboard/ConfigOptions.vue

@@ -8,6 +8,7 @@ import ThresholdTab from "@/components/dashboard/tabs/ThresholdTab.vue";
 import ContoursTab from "@/components/dashboard/tabs/ContoursTab.vue";
 import AprilTagTab from "@/components/dashboard/tabs/AprilTagTab.vue";
 import ArucoTab from "@/components/dashboard/tabs/ArucoTab.vue";
+import ObjectDetectionTab from "@/components/dashboard/tabs/ObjectDetectionTab.vue";
 import OutputTab from "@/components/dashboard/tabs/OutputTab.vue";
 import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue";
 import PnPTab from "@/components/dashboard/tabs/PnPTab.vue";
@@ -40,6 +41,10 @@ const allTabs = Object.freeze({
     tabName: "Aruco",
     component: ArucoTab
   },
+  objectDetectionTab: {
+    tabName: "Object Detection",
+    component: ObjectDetectionTab
+  },
   outputTab: {
     tabName: "Output",
     component: OutputTab
@@ -75,6 +80,7 @@ const getTabGroups = (): ConfigOption[][] => {
         allTabs.contoursTab,
         allTabs.apriltagTab,
         allTabs.arucoTab,
+        allTabs.objectDetectionTab,
         allTabs.outputTab
       ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
@@ -82,14 +88,21 @@ const getTabGroups = (): ConfigOption[][] => {
   } else if (lgAndDown) {
     return [
       [allTabs.inputTab],
-      [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [
+        allTabs.thresholdTab,
+        allTabs.contoursTab,
+        allTabs.apriltagTab,
+        allTabs.arucoTab,
+        allTabs.objectDetectionTab,
+        allTabs.outputTab
+      ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   } else if (xl) {
     return [
       [allTabs.inputTab],
       [allTabs.thresholdTab],
-      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.objectDetectionTab, allTabs.outputTab],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   }
@@ -103,17 +116,20 @@ const tabGroups = computed<ConfigOption[][]>(() => {
   const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled;
   const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag;
   const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
+  const isObjectDetection =
+    useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection;
 
   return getTabGroups()
     .map((tabGroup) =>
       tabGroup.filter(
         (tabConfig) =>
           !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
-          !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
+          !((!allow3d || isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
           !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
-          !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco
+          !(!isAruco && tabConfig.tabName === "Aruco") &&
+          !(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out aruco unless we actually are doing Aruco
       )
     )
     .filter((it) => it.length); // Remove empty tab groups

photon-client/src/components/dashboard/StreamConfigCard.vue

@@ -39,11 +39,11 @@ const processingMode = computed<number>({
         <p style="color: white">Processing Mode</p>
         <v-btn-toggle v-model="processingMode" mandatory dark class="fill">
           <v-btn color="secondary">
-            <v-icon>mdi-square-outline</v-icon>
+            <v-icon left>mdi-square-outline</v-icon>
             <span>2D</span>
           </v-btn>
           <v-btn color="secondary" :disabled="!useCameraSettingsStore().isCurrentVideoFormatCalibrated">
-            <v-icon>mdi-cube-outline</v-icon>
+            <v-icon left>mdi-cube-outline</v-icon>
             <span>3D</span>
           </v-btn>
         </v-btn-toggle>
@@ -54,12 +54,12 @@ const processingMode = computed<number>({
         <p style="color: white">Stream Display</p>
         <v-btn-toggle v-model="localValue" :multiple="true" mandatory dark class="fill">
           <v-btn color="secondary" class="fill">
-            <v-icon>mdi-import</v-icon>
+            <v-icon left class="mode-btn-icon">mdi-import</v-icon>
-            <span>Raw</span>
+            <span class="mode-btn-label">Raw</span>
           </v-btn>
           <v-btn color="secondary" class="fill">
-            <v-icon>mdi-export</v-icon>
+            <v-icon left class="mode-btn-icon">mdi-export</v-icon>
-            <span>Processed</span>
+            <span class="mode-btn-label">Processed</span>
           </v-btn>
         </v-btn-toggle>
       </v-col>
@@ -82,4 +82,13 @@ th {
   width: 80px;
   text-align: center;
 }
+
+@media only screen and (max-width: 351px) {
+  .mode-btn-icon {
+    margin: 0 !important;
+  }
+  .mode-btn-label {
+    display: none;
+  }
+}
 </style>

photon-client/src/components/dashboard/tabs/AprilTagTab.vue

@@ -1,23 +1,25 @@
 <script setup lang="ts">
-import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { PipelineType } from "@/types/PipelineTypes";
 import PvSelect from "@/components/common/pv-select.vue";
 import PvSlider from "@/components/common/pv-slider.vue";
 import PvSwitch from "@/components/common/pv-switch.vue";
 import { computed, getCurrentInstance } from "vue";
 import { useStateStore } from "@/stores/StateStore";
+import type { ActivePipelineSettings } from "@/types/PipelineTypes";
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 
 // TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
 // Defer reference to store access method
-const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+const currentPipelineSettings = computed<ActivePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings
+);
 
-const interactiveCols = computed(
+const interactiveCols = computed(() =>
-  () =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
-    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
-    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
-)
+    : 8
-  ? 9
+);
-  : 8;
 </script>
 
 <template>

photon-client/src/components/dashboard/tabs/ArucoTab.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { PipelineType } from "@/types/PipelineTypes";
+import { PipelineType, type ActivePipelineSettings } from "@/types/PipelineTypes";
 import PvSlider from "@/components/common/pv-slider.vue";
 import PvSwitch from "@/components/common/pv-switch.vue";
 import PvRangeSlider from "@/components/common/pv-range-slider.vue";
@@ -10,15 +10,16 @@ import { useStateStore } from "@/stores/StateStore";
 
 // TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
 // Defer reference to store access method
-const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+const currentPipelineSettings = computed<ActivePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings
+);
 
-const interactiveCols = computed(
+const interactiveCols = computed(() =>
-  () =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
-    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
-    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
-)
+    : 8
-  ? 9
+);
-  : 8;
 </script>
 
 <template>

photon-client/src/components/dashboard/tabs/ContoursTab.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { PipelineType } from "@/types/PipelineTypes";
+import { type ActivePipelineSettings, PipelineType } from "@/types/PipelineTypes";
 import PvRangeSlider from "@/components/common/pv-range-slider.vue";
 import PvSelect from "@/components/common/pv-select.vue";
 import PvSlider from "@/components/common/pv-slider.vue";
@@ -9,7 +9,9 @@ import { useStateStore } from "@/stores/StateStore";
 
 // TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
 // Defer reference to store access method
-const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+const currentPipelineSettings = computed<ActivePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings
+);
 
 // TODO fix pv-range-slider so that store access doesn't need to be deferred
 const contourArea = computed<[number, number]>({
@@ -26,34 +28,33 @@ const contourFullness = computed<[number, number]>({
 });
 const contourPerimeter = computed<[number, number]>({
   get: () =>
-    currentPipelineSettings.pipelineType === PipelineType.ColoredShape
+    currentPipelineSettings.value.pipelineType === PipelineType.ColoredShape
-      ? (Object.values(currentPipelineSettings.contourPerimeter) as [number, number])
+      ? (Object.values(currentPipelineSettings.value.contourPerimeter) as [number, number])
       : ([0, 0] as [number, number]),
   set: (v) => {
-    if (currentPipelineSettings.pipelineType === PipelineType.ColoredShape) {
+    if (currentPipelineSettings.value.pipelineType === PipelineType.ColoredShape) {
-      currentPipelineSettings.contourPerimeter = v;
+      currentPipelineSettings.value.contourPerimeter = v;
     }
   }
 });
 const contourRadius = computed<[number, number]>({
   get: () =>
-    currentPipelineSettings.pipelineType === PipelineType.ColoredShape
+    currentPipelineSettings.value.pipelineType === PipelineType.ColoredShape
-      ? (Object.values(currentPipelineSettings.contourRadius) as [number, number])
+      ? (Object.values(currentPipelineSettings.value.contourRadius) as [number, number])
       : ([0, 0] as [number, number]),
   set: (v) => {
-    if (currentPipelineSettings.pipelineType === PipelineType.ColoredShape) {
+    if (currentPipelineSettings.value.pipelineType === PipelineType.ColoredShape) {
-      currentPipelineSettings.contourRadius = v;
+      currentPipelineSettings.value.contourRadius = v;
     }
   }
 });
 
-const interactiveCols = computed(
+const interactiveCols = computed(() =>
-  () =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
-    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
-    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
-)
+    : 8
-  ? 9
+);
-  : 8;
 </script>
 
 <template>
@@ -103,8 +104,8 @@ const interactiveCols = computed(
       v-model="contourPerimeter"
       label="Perimeter"
       tooltip="Min and max perimeter of the shape, in pixels"
-      min="0"
+      :min="0"
-      max="4000"
+      :max="4000"
       :slider-cols="interactiveCols"
       @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourPerimeter: value }, false)"
     />

photon-client/src/components/dashboard/tabs/InputTab.vue

@@ -6,13 +6,14 @@ import PvSelect from "@/components/common/pv-select.vue";
 import { computed, getCurrentInstance } from "vue";
 import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 import { useStateStore } from "@/stores/StateStore";
+import { getResolutionString } from "@/lib/PhotonUtils";
 
 // Due to something with libcamera or something else IDK much about, the 90° rotations need to be disabled if the libcamera drivers are being used.
 const cameraRotations = computed(() =>
   ["Normal", "90° CW", "180°", "90° CCW"].map((v, i) => ({
     name: v,
     value: i,
-    disabled: useSettingsStore().gpuAccelerationEnabled ? [1, 3].includes(i) : false
+    disabled: useCameraSettingsStore().isCSICamera ? [1, 3].includes(i) : false
   }))
 );
 
@@ -30,7 +31,7 @@ const getNumberOfSkippedDivisors = () => streamDivisors.length - getFilteredStre
 
 const cameraResolutions = computed(() =>
   useCameraSettingsStore().currentCameraSettings.validVideoFormats.map(
-    (f) => `${f.resolution.width} X ${f.resolution.height} at ${f.fps} FPS, ${f.pixelFormat}`
+    (f) => `${getResolutionString(f.resolution)} at ${f.fps} FPS, ${f.pixelFormat}`
   )
 );
 const handleResolutionChange = (value: number) => {
@@ -48,7 +49,11 @@ const streamResolutions = computed(() => {
   const streamDivisors = getFilteredStreamDivisors();
   const currentResolution = useCameraSettingsStore().currentVideoFormat.resolution;
   return streamDivisors.map(
-    (x) => `${Math.floor(currentResolution.width / x)} X ${Math.floor(currentResolution.height / x)}`
+    (x) =>
+      `${getResolutionString({
+        width: Math.floor(currentResolution.width / x),
+        height: Math.floor(currentResolution.height / x)
+      })}`
   );
 });
 const handleStreamResolutionChange = (value: number) => {
@@ -58,13 +63,12 @@ const handleStreamResolutionChange = (value: number) => {
   );
 };
 
-const interactiveCols = computed(
+const interactiveCols = computed(() =>
-  () =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
-    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
-    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
-)
+    : 8
-  ? 9
+);
-  : 8;
 </script>
 
 <template>

photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue

@@ -0,0 +1,83 @@
+<script setup lang="ts">
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { type ActivePipelineSettings, PipelineType } from "@/types/PipelineTypes";
+import PvSlider from "@/components/common/pv-slider.vue";
+import { computed, getCurrentInstance } from "vue";
+import { useStateStore } from "@/stores/StateStore";
+
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = computed<ActivePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings
+);
+
+// TODO fix pv-range-slider so that store access doesn't need to be deferred
+const contourArea = computed<[number, number]>({
+  get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourArea) as [number, number],
+  set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourArea = v)
+});
+const contourRatio = computed<[number, number]>({
+  get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourRatio) as [number, number],
+  set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourRatio = v)
+});
+
+const interactiveCols = computed(() =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
+    : 8
+);
+</script>
+
+<template>
+  <div v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+    <pv-slider
+      v-model="currentPipelineSettings.confidence"
+      class="pt-2"
+      :slider-cols="interactiveCols"
+      label="Confidence"
+      tooltip="The minimum confidence for a detection to be considered valid. Bigger numbers mean fewer but more probable detections are allowed through."
+      :min="0"
+      :max="1"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
+    />
+    <pv-range-slider
+      v-model="contourArea"
+      label="Area"
+      :min="0"
+      :max="100"
+      :slider-cols="interactiveCols"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourArea: value }, false)"
+    />
+    <pv-range-slider
+      v-model="contourRatio"
+      label="Ratio (W/H)"
+      tooltip="Min and max ratio between the width and height of a contour's bounding rectangle"
+      :min="0"
+      :max="100"
+      :slider-cols="interactiveCols"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourRatio: value }, false)"
+    />
+    <pv-select
+      v-model="useCameraSettingsStore().currentPipelineSettings.contourTargetOrientation"
+      label="Target Orientation"
+      tooltip="Used to determine how to calculate target landmarks, as well as aspect ratio"
+      :items="['Portrait', 'Landscape']"
+      :select-cols="interactiveCols"
+      @input="
+        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourTargetOrientation: value }, false)
+      "
+    />
+    <pv-select
+      v-model="currentPipelineSettings.contourSortMode"
+      label="Target Sort"
+      tooltip="Chooses the sorting mode used to determine the 'best' targets to provide to user code"
+      :select-cols="interactiveCols"
+      :items="['Largest', 'Smallest', 'Highest', 'Lowest', 'Rightmost', 'Leftmost', 'Centermost']"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourSortMode: value }, false)"
+    />
+  </div>
+</template>

photon-client/src/components/dashboard/tabs/OutputTab.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import PvSelect from "@/components/common/pv-select.vue";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { PipelineType, RobotOffsetPointMode } from "@/types/PipelineTypes";
+import { type ActivePipelineSettings, PipelineType, RobotOffsetPointMode } from "@/types/PipelineTypes";
 import PvSwitch from "@/components/common/pv-switch.vue";
 import { computed, getCurrentInstance } from "vue";
 import { RobotOffsetType } from "@/types/SettingTypes";
@@ -40,15 +40,18 @@ const offsetPoints = computed<MetricItem[]>(() => {
   }
 });
 
-const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = computed<ActivePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings
+);
 
-const interactiveCols = computed(
+const interactiveCols = computed(() =>
-  () =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
-    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
-    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
-)
+    : 8
-  ? 9
+);
-  : 8;
 </script>
 
 <template>

photon-client/src/components/dashboard/tabs/PnPTab.vue

@@ -6,13 +6,12 @@ import PvSlider from "@/components/common/pv-slider.vue";
 import { computed, getCurrentInstance } from "vue";
 import { useStateStore } from "@/stores/StateStore";
 
-const interactiveCols = computed(
+const interactiveCols = computed(() =>
-  () =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
-    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
-    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
-)
+    : 8
-  ? 9
+);
-  : 8;
 </script>
 
 <template>

photon-client/src/components/dashboard/tabs/TargetsTab.vue

@@ -1,134 +1,298 @@
 <script setup lang="ts">
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { PipelineType } from "@/types/PipelineTypes";
+import { type ActivePipelineSettings, PipelineType } from "@/types/PipelineTypes";
 import { useStateStore } from "@/stores/StateStore";
+import { angleModulus, toDeg } from "@/lib/MathUtils";
+import { computed } from "vue";
 
-const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = computed<ActivePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings
+);
+
+const calculateStdDev = (values: number[]): number => {
+  if (values.length < 2) return 0;
+
+  // Use mean of cosine/sine components to handle angle wrapping
+  const cosines = values.map((it) => Math.cos(it));
+  const sines = values.map((it) => Math.sin(it));
+  const cosmean = cosines.reduce((sum, number) => sum + number, 0) / values.length;
+  const sinmean = sines.reduce((sum, number) => sum + number, 0) / values.length;
+
+  // Borrowed from WPILib's Rotation2d
+  const hypot = Math.hypot(cosmean, sinmean);
+  const mean = hypot > 1e-6 ? Math.atan2(sinmean / hypot, cosmean / hypot) : 0;
+
+  return Math.sqrt(values.map((x) => Math.pow(angleModulus(x - mean), 2)).reduce((a, b) => a + b) / values.length);
+};
+const resetCurrentBuffer = () => {
+  // Need to clear the array in place
+  while (useStateStore().currentMultitagBuffer?.length != 0) useStateStore().currentMultitagBuffer?.pop();
+};
 </script>
 
 <template>
   <div>
-    <v-row align="start" class="pb-4" style="height: 300px">
+    <v-row align="start" class="pb-4">
-      <!-- Simple table height must be set here and in the CSS for the fixed-header to work -->
+      <v-simple-table dense class="pt-2 pb-12">
-      <v-simple-table fixed-header dense dark>
         <template #default>
-          <thead style="font-size: 1.25rem">
+          <thead>
             <tr>
               <th
                 v-if="
-                  useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+                  currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
-                  useCameraSettingsStore().currentPipelineType === PipelineType.Aruco
+                  currentPipelineSettings.pipelineType === PipelineType.Aruco
                 "
-                class="text-center"
+                class="text-center white--text"
               >
                 Fiducial ID
               </th>
+              <template v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+                <th class="text-center white--text">Class</th>
+                <th class="text-center white--text">Confidence</th>
+              </template>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
-                <th class="text-center">Pitch &theta;&deg;</th>
+                <th class="text-center white--text">Pitch &theta;&deg;</th>
-                <th class="text-center">Yaw &theta;&deg;</th>
+                <th class="text-center white--text">Yaw &theta;&deg;</th>
-                <th class="text-center">Skew &theta;&deg;</th>
+                <th class="text-center white--text">Skew &theta;&deg;</th>
-                <th class="text-center">Area %</th>
+                <th class="text-center white--text">Area %</th>
               </template>
               <template v-else>
-                <th class="text-center">X meters</th>
+                <th class="text-center white--text">X meters</th>
-                <th class="text-center">Y meters</th>
+                <th class="text-center white--text">Y meters</th>
-                <th class="text-center">Z Angle &theta;&deg;</th>
+                <th class="text-center white--text">Z Angle &theta;&deg;</th>
               </template>
               <template
                 v-if="
-                  (useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+                  (currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
-                    useCameraSettingsStore().currentPipelineType === PipelineType.Aruco) &&
+                    currentPipelineSettings.pipelineType === PipelineType.Aruco) &&
                   useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
                 "
               >
-                <th class="text-center">Ambiguity %</th>
+                <th class="text-center white--text">Ambiguity Ratio</th>
               </template>
             </tr>
           </thead>
           <tbody>
-            <tr v-for="(target, index) in useStateStore().currentPipelineResults?.targets" :key="index">
+            <tr
+              v-for="(target, index) in useStateStore().currentPipelineResults?.targets"
+              :key="index"
+              class="white--text"
+            >
               <td
                 v-if="
-                  useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+                  currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
-                  useCameraSettingsStore().currentPipelineType === PipelineType.Aruco
+                  currentPipelineSettings.pipelineType === PipelineType.Aruco
                 "
+                class="text-center"
               >
                 {{ target.fiducialId }}
               </td>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ useStateStore().currentPipelineResults?.classNames[target.classId] }}
+              </td>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ target.confidence.toFixed(2) }}
+              </td>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
-                <td>{{ target.pitch.toFixed(2) }}&deg;</td>
+                <td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
-                <td>{{ target.yaw.toFixed(2) }}&deg;</td>
+                <td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>
-                <td>{{ target.skew.toFixed(2) }}&deg;</td>
+                <td class="text-center">{{ target.skew.toFixed(2) }}&deg;</td>
-                <td>{{ target.area.toFixed(2) }}&deg;</td>
+                <td class="text-center">{{ target.area.toFixed(2) }}&deg;</td>
               </template>
               <template v-else>
-                <td>{{ target.pose?.x.toFixed(2) }}&nbsp;m</td>
+                <td class="text-center">{{ target.pose?.x.toFixed(2) }}&nbsp;m</td>
-                <td>{{ target.pose?.y.toFixed(2) }}&nbsp;m</td>
+                <td class="text-center">{{ target.pose?.y.toFixed(2) }}&nbsp;m</td>
-                <td>{{ (((target.pose?.angle_z || 0) * 180.0) / Math.PI).toFixed(2) }}&deg;</td>
+                <td class="text-center">{{ toDeg(target.pose?.angle_z || 0).toFixed(2) }}&deg;</td>
               </template>
               <template
                 v-if="
-                  (useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+                  (currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
-                    useCameraSettingsStore().currentPipelineType === PipelineType.Aruco) &&
+                    currentPipelineSettings.pipelineType === PipelineType.Aruco) &&
                   useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
                 "
               >
-                <td>{{ target.ambiguity >= 0 ? target.ambiguity?.toFixed(2) + "%" : "(In Multi-Target)" }}</td>
+                <td class="text-center">
+                  {{ target.ambiguity >= 0 ? target.ambiguity.toFixed(2) : "(In Multi-Target)" }}
+                </td>
               </template>
             </tr>
           </tbody>
         </template>
       </v-simple-table>
     </v-row>
-    <v-row
+    <v-container
       v-if="
-        (useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
+        (currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
-          useCameraSettingsStore().currentPipelineType === PipelineType.Aruco) &&
+          currentPipelineSettings.pipelineType === PipelineType.Aruco) &&
         currentPipelineSettings.doMultiTarget &&
         useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
         useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
       "
-      align="start"
-      class="pb-4 white--text"
     >
-      <v-card-subtitle class="ma-0 pa-0 pb-4" style="font-size: 16px">Multi-tag pose, field-to-camera</v-card-subtitle>
+      <v-row class="pb-4 white--text">
-      <v-simple-table fixed-header height="100%" dense dark>
+        <v-card-subtitle class="ma-0 pa-0 pb-4" style="font-size: 16px"
-        <thead style="font-size: 1.25rem">
+          >Multi-tag pose, field-to-camera</v-card-subtitle
-          <th class="text-center">X meters</th>
+        >
-          <th class="text-center">Y meters</th>
+        <v-simple-table dense>
-          <th class="text-center">Z Angle &theta;&deg;</th>
+          <template #default>
-          <th class="text-center">Tags</th>
+            <thead>
-        </thead>
+              <tr class="white--text">
-        <tbody v-show="useStateStore().currentPipelineResults?.multitagResult">
+                <th class="text-center white--text">X meters</th>
-          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.x.toFixed(2) }}&nbsp;m</td>
+                <th class="text-center white--text">Y meters</th>
-          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.y.toFixed(2) }}&nbsp;m</td>
+                <th class="text-center white--text">Z meters</th>
-          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.angle_z.toFixed(2) }}&deg;</td>
+                <th class="text-center white--text">X Angle &theta;&deg;</th>
-          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.fiducialIDsUsed }}</td>
+                <th class="text-center white--text">Y Angle &theta;&deg;</th>
-        </tbody>
+                <th class="text-center white--text">Z Angle &theta;&deg;</th>
-      </v-simple-table>
+                <th class="text-center white--text">Tags</th>
-    </v-row>
+              </tr>
+            </thead>
+            <tbody v-show="useStateStore().currentPipelineResults?.multitagResult">
+              <tr>
+                <td class="text-center white--text">
+                  {{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.x.toFixed(2) }}&nbsp;m
+                </td>
+                <td class="text-center white--text">
+                  {{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.y.toFixed(2) }}&nbsp;m
+                </td>
+                <td class="text-center white--text">
+                  {{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.z.toFixed(2) }}&nbsp;m
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    toDeg(useStateStore().currentPipelineResults?.multitagResult?.bestTransform.angle_x || 0).toFixed(
+                      2
+                    )
+                  }}&deg;
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    toDeg(useStateStore().currentPipelineResults?.multitagResult?.bestTransform.angle_y || 0).toFixed(
+                      2
+                    )
+                  }}&deg;
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    toDeg(useStateStore().currentPipelineResults?.multitagResult?.bestTransform.angle_z || 0).toFixed(
+                      2
+                    )
+                  }}&deg;
+                </td>
+                <td class="text-center white--text">
+                  {{ useStateStore().currentPipelineResults?.multitagResult?.fiducialIDsUsed }}
+                </td>
+              </tr>
+            </tbody>
+          </template>
+        </v-simple-table>
+      </v-row>
+      <v-row class="pb-4 white--text" style="display: flex; flex-direction: column">
+        <v-card-subtitle class="ma-0 pa-0 pb-4 pr-4" style="font-size: 16px"
+          >Multi-tag pose standard deviation over the last
+          {{ useStateStore().currentMultitagBuffer?.length || "NaN" }}/100 samples
+        </v-card-subtitle>
+        <v-btn color="secondary" class="mb-4 mt-1" style="width: min-content" depressed @click="resetCurrentBuffer"
+          >Reset Samples</v-btn
+        >
+        <v-simple-table dense>
+          <template #default>
+            <thead>
+              <tr>
+                <th class="text-center white--text">X meters</th>
+                <th class="text-center white--text">Y meters</th>
+                <th class="text-center white--text">Z meters</th>
+                <th class="text-center white--text">X Angle &theta;&deg;</th>
+                <th class="text-center white--text">Y Angle &theta;&deg;</th>
+                <th class="text-center white--text">Z Angle &theta;&deg;</th>
+              </tr>
+            </thead>
+            <tbody v-show="useStateStore().currentPipelineResults?.multitagResult">
+              <tr>
+                <td class="text-center white--text">
+                  {{
+                    calculateStdDev(useStateStore().currentMultitagBuffer?.map((v) => v.bestTransform.x) || []).toFixed(
+                      5
+                    )
+                  }}&nbsp;m
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    calculateStdDev(useStateStore().currentMultitagBuffer?.map((v) => v.bestTransform.y) || []).toFixed(
+                      5
+                    )
+                  }}&nbsp;m
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    calculateStdDev(useStateStore().currentMultitagBuffer?.map((v) => v.bestTransform.z) || []).toFixed(
+                      5
+                    )
+                  }}&nbsp;m
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    calculateStdDev(
+                      useStateStore().currentMultitagBuffer?.map((v) => toDeg(v.bestTransform.angle_x)) || []
+                    ).toFixed(5)
+                  }}&deg;
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    calculateStdDev(
+                      useStateStore().currentMultitagBuffer?.map((v) => toDeg(v.bestTransform.angle_y)) || []
+                    ).toFixed(5)
+                  }}&deg;
+                </td>
+                <td class="text-center white--text">
+                  {{
+                    calculateStdDev(
+                      useStateStore().currentMultitagBuffer?.map((v) => toDeg(v.bestTransform.angle_z)) || []
+                    ).toFixed(5)
+                  }}&deg;
+                </td>
+              </tr>
+            </tbody>
+          </template>
+        </v-simple-table>
+      </v-row>
+    </v-container>
   </div>
 </template>
 
 <style scoped lang="scss">
 .v-data-table {
-  width: 100%;
-  height: 100%;
-  text-align: center;
   background-color: #006492 !important;
+  width: 100%;
+  font-size: 1rem !important;
 
-  th,
+  thead {
-  td {
+    tr {
-    background-color: #006492 !important;
+      th {
-    font-size: 1rem !important;
+        font-size: 1rem !important;
+        color: white !important;
+      }
+    }
   }
-
+  tbody {
-  td {
+    :hover {
-    font-family: monospace !important;
+      td {
-  }
+        background-color: #005281 !important;
-
+      }
-  tbody :hover td {
+    }
-    background-color: #005281 !important;
+    tr {
+      td {
+        font-size: 1rem !important;
+        color: white !important;
+      }
+    }
   }
 
   ::-webkit-scrollbar {

photon-client/src/components/dashboard/tabs/ThresholdTab.vue

@@ -124,13 +124,12 @@ onBeforeUnmount(() => {
   cameraStream.removeEventListener("click", handleStreamClick);
 });
 
-const interactiveCols = computed(
+const interactiveCols = computed(() =>
-  () =>
+  (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
-    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+  (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
-    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+    ? 9
-)
+    : 8
-  ? 9
+);
-  : 8;
 </script>
 
 <template>

photon-client/src/components/settings/ApriltagControlCard.vue

@@ -2,15 +2,16 @@
 import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 import { Euler, Quaternion as ThreeQuat } from "three";
 import type { Quaternion } from "@/types/PhotonTrackingTypes";
+import { toDeg } from "@/lib/MathUtils";
 
 const quaternionToEuler = (rot_quat: Quaternion): { x: number; y: number; z: number } => {
   const quat = new ThreeQuat(rot_quat.X, rot_quat.Y, rot_quat.Z, rot_quat.W);
   const euler = new Euler().setFromQuaternion(quat, "ZYX");
 
   return {
-    x: euler.x * (180.0 / Math.PI),
+    x: toDeg(euler.x),
-    y: euler.y * (180.0 / Math.PI),
+    y: toDeg(euler.y),
-    z: euler.z * (180.0 / Math.PI)
+    z: toDeg(euler.z)
   };
 };
 </script>
@@ -62,6 +63,7 @@ const quaternionToEuler = (rot_quat: Quaternion): { x: number; y: number; z: num
   td {
     background-color: #006492 !important;
     font-size: 1rem !important;
+    color: white !important;
   }
 
   td {

photon-client/src/components/settings/DeviceControlCard.vue

@@ -63,11 +63,12 @@ const offlineUpdate = ref();
 const openOfflineUpdatePrompt = () => {
   offlineUpdate.value.click();
 };
-const handleOfflineUpdate = (payload: Event & { target: (EventTarget & HTMLInputElement) | null }) => {
+const handleOfflineUpdate = () => {
-  if (payload.target === null || !payload.target.files) return;
+  const files = offlineUpdate.value.files;
+  if (files.length === 0) return;
 
   const formData = new FormData();
-  formData.append("jarData", payload.target.files[0]);
+  formData.append("jarData", files[0]);
 
   useStateStore().showSnackbarMessage({
     message: "New Software Upload in Progress...",
@@ -209,20 +210,20 @@ const handleSettingsImport = () => {
       <v-row>
         <v-col cols="12" lg="4" md="6">
           <v-btn color="red" @click="restartProgram">
-            <v-icon left> mdi-restart </v-icon>
+            <v-icon left class="open-icon"> mdi-restart </v-icon>
-            Restart PhotonVision
+            <span class="open-label">Restart PhotonVision</span>
           </v-btn>
         </v-col>
         <v-col cols="12" lg="4" md="6">
           <v-btn color="red" @click="restartDevice">
-            <v-icon left> mdi-restart-alert </v-icon>
+            <v-icon left class="open-icon"> mdi-restart-alert </v-icon>
-            Restart Device
+            <span class="open-label">Restart Device</span>
           </v-btn>
         </v-col>
         <v-col cols="12" lg="4">
           <v-btn color="secondary" @click="openOfflineUpdatePrompt">
-            <v-icon left> mdi-upload </v-icon>
+            <v-icon left class="open-icon"> mdi-upload </v-icon>
-            Offline Update
+            <span class="open-label">Offline Update</span>
           </v-btn>
           <input ref="offlineUpdate" type="file" accept=".jar" style="display: none" @change="handleOfflineUpdate" />
         </v-col>
@@ -231,8 +232,8 @@ const handleSettingsImport = () => {
       <v-row>
         <v-col cols="12" sm="6">
           <v-btn color="secondary" @click="() => (showImportDialog = true)">
-            <v-icon left> mdi-import </v-icon>
+            <v-icon left class="open-icon"> mdi-import </v-icon>
-            Import Settings
+            <span class="open-label">Import Settings</span>
           </v-btn>
           <v-dialog
             v-model="showImportDialog"
@@ -278,8 +279,8 @@ const handleSettingsImport = () => {
                   align="center"
                 >
                   <v-btn color="secondary" :disabled="importFile === null" @click="handleSettingsImport">
-                    <v-icon left> mdi-import </v-icon>
+                    <v-icon left class="open-icon"> mdi-import </v-icon>
-                    Import Settings
+                    <span class="open-label">Import Settings</span>
                   </v-btn>
                 </v-row>
               </v-card-text>
@@ -288,8 +289,8 @@ const handleSettingsImport = () => {
         </v-col>
         <v-col cols="12" sm="6">
           <v-btn color="secondary" @click="openExportSettingsPrompt">
-            <v-icon left> mdi-export </v-icon>
+            <v-icon left class="open-icon"> mdi-export </v-icon>
-            Export Settings
+            <span class="open-label">Export Settings</span>
           </v-btn>
           <a
             ref="exportSettings"
@@ -301,8 +302,8 @@ const handleSettingsImport = () => {
         </v-col>
         <v-col cols="12" sm="6">
           <v-btn color="secondary" @click="openExportLogsPrompt">
-            <v-icon left> mdi-download </v-icon>
+            <v-icon left class="open-icon"> mdi-download </v-icon>
-            Download Current Log
+            <span class="open-label">Download Current Log</span>
 
             <!-- Special hidden link that gets 'clicked' when the user exports journalctl logs -->
             <a
@@ -316,8 +317,8 @@ const handleSettingsImport = () => {
         </v-col>
         <v-col cols="12" sm="6">
           <v-btn color="secondary" @click="useStateStore().showLogModal = true">
-            <v-icon left> mdi-eye </v-icon>
+            <v-icon left class="open-icon"> mdi-eye </v-icon>
-            Show log viewer
+            <span class="open-label">Show log viewer</span>
           </v-btn>
         </v-col>
       </v-row>
@@ -332,4 +333,12 @@ const handleSettingsImport = () => {
 .v-btn {
   width: 100%;
 }
+@media only screen and (max-width: 351px) {
+  .open-icon {
+    margin: 0 !important;
+  }
+  .open-label {
+    display: none;
+  }
+}
 </style>

photon-client/src/components/settings/NetworkingCard.vue

@@ -1,14 +1,21 @@
 <script setup lang="ts">
 import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
-import { computed, ref } from "vue";
+import { computed, ref, watchEffect } from "vue";
 import PvInput from "@/components/common/pv-input.vue";
 import PvRadio from "@/components/common/pv-radio.vue";
 import PvSwitch from "@/components/common/pv-switch.vue";
 import PvSelect from "@/components/common/pv-select.vue";
-import { NetworkConnectionType } from "@/types/SettingTypes";
+import { type ConfigurableNetworkSettings, NetworkConnectionType } from "@/types/SettingTypes";
 import { useStateStore } from "@/stores/StateStore";
 
+// Copy object to remove reference to store
+const tempSettingsStruct = ref<ConfigurableNetworkSettings>(Object.assign({}, useSettingsStore().network));
+const resetTempSettingsStruct = () => {
+  tempSettingsStruct.value = Object.assign({}, useSettingsStore().network);
+};
+
 const settingsValid = ref(true);
+
 const isValidNetworkTablesIP = (v: string | undefined): boolean => {
   // Check if it is a valid team number between 1-9999
   const teamNumberRegex = /^[1-9][0-9]{0,3}$/;
@@ -38,18 +45,57 @@ const isValidHostname = (v: string | undefined) => {
   return hostnameRegex.test(v);
 };
 
+const settingsHaveChanged = (): boolean => {
+  const a = useSettingsStore().network;
+  const b = tempSettingsStruct.value;
+
+  return (
+    a.ntServerAddress !== b.ntServerAddress ||
+    a.connectionType !== b.connectionType ||
+    a.staticIp !== b.staticIp ||
+    a.hostname !== b.hostname ||
+    a.runNTServer !== b.runNTServer ||
+    a.shouldManage !== b.shouldManage ||
+    a.shouldPublishProto !== b.shouldPublishProto ||
+    a.networkManagerIface !== b.networkManagerIface ||
+    a.setStaticCommand !== b.setStaticCommand ||
+    a.setDHCPcommand !== b.setDHCPcommand
+  );
+};
+
 const saveGeneralSettings = () => {
   const changingStaticIp = useSettingsStore().network.connectionType === NetworkConnectionType.Static;
 
+  // replace undefined members with empty strings for backend
+  const payload = {
+    connectionType: tempSettingsStruct.value.connectionType,
+    hostname: tempSettingsStruct.value.hostname,
+    networkManagerIface: tempSettingsStruct.value.networkManagerIface || "",
+    ntServerAddress: tempSettingsStruct.value.ntServerAddress,
+    runNTServer: tempSettingsStruct.value.runNTServer,
+    setDHCPcommand: tempSettingsStruct.value.setDHCPcommand || "",
+    setStaticCommand: tempSettingsStruct.value.setStaticCommand || "",
+    shouldManage: tempSettingsStruct.value.shouldManage,
+    shouldPublishProto: tempSettingsStruct.value.shouldPublishProto,
+    staticIp: tempSettingsStruct.value.staticIp
+  };
+
   useSettingsStore()
-    .saveGeneralSettings()
+    .updateGeneralSettings(payload)
     .then((response) => {
       useStateStore().showSnackbarMessage({
         message: response.data.text || response.data,
         color: "success"
       });
+
+      // Update the local settings cause the backend checked their validity. Assign is to deref value
+      useSettingsStore().network = {
+        ...useSettingsStore().network,
+        ...Object.assign({}, tempSettingsStruct.value)
+      };
     })
     .catch((error) => {
+      resetTempSettingsStruct();
       if (error.response) {
         if (error.status === 504 || changingStaticIp) {
           useStateStore().showSnackbarMessage({
@@ -80,7 +126,12 @@ const saveGeneralSettings = () => {
 
 const currentNetworkInterfaceIndex = computed<number>({
   get: () => useSettingsStore().networkInterfaceNames.indexOf(useSettingsStore().network.networkManagerIface || ""),
-  set: (v) => (useSettingsStore().network.networkManagerIface = useSettingsStore().networkInterfaceNames[v])
+  set: (v) => (tempSettingsStruct.value.networkManagerIface = useSettingsStore().networkInterfaceNames[v])
+});
+
+watchEffect(() => {
+  // Reset temp settings on remote network settings change
+  resetTempSettingsStruct();
 });
 </script>
 
@@ -90,11 +141,11 @@ const currentNetworkInterfaceIndex = computed<number>({
     <div class="ml-5">
       <v-form ref="form" v-model="settingsValid">
         <pv-input
-          v-model="useSettingsStore().network.ntServerAddress"
+          v-model="tempSettingsStruct.ntServerAddress"
           label="Team Number/NetworkTables Server Address"
           tooltip="Enter the Team Number or the IP address of the NetworkTables Server"
           :label-cols="4"
-          :disabled="useSettingsStore().network.runNTServer"
+          :disabled="tempSettingsStruct.runNTServer"
           :rules="[
             (v) =>
               isValidNetworkTablesIP(v) ||
@@ -102,10 +153,7 @@ const currentNetworkInterfaceIndex = computed<number>({
           ]"
         />
         <v-banner
-          v-show="
+          v-show="!isValidNetworkTablesIP(tempSettingsStruct.ntServerAddress) && !tempSettingsStruct.runNTServer"
-            !isValidNetworkTablesIP(useSettingsStore().network.ntServerAddress) &&
-            !useSettingsStore().network.runNTServer
-          "
           rounded
           color="red"
           text-color="white"
@@ -115,42 +163,63 @@ const currentNetworkInterfaceIndex = computed<number>({
           The NetworkTables Server Address is not set or is invalid. NetworkTables is unable to connect.
         </v-banner>
         <pv-radio
-          v-model="useSettingsStore().network.connectionType"
+          v-show="!useSettingsStore().network.networkingDisabled"
+          v-model="tempSettingsStruct.connectionType"
           label="IP Assignment Mode"
           tooltip="DHCP will make the radio (router) automatically assign an IP address; this may result in an IP address that changes across reboots. Static IP assignment means that you pick the IP address and it won't change."
           :input-cols="12 - 4"
           :list="['DHCP', 'Static']"
-          :disabled="!(useSettingsStore().network.shouldManage && useSettingsStore().network.canManage)"
+          :disabled="
+            !tempSettingsStruct.shouldManage ||
+            !useSettingsStore().network.canManage ||
+            useSettingsStore().network.networkingDisabled
+          "
         />
         <pv-input
-          v-if="useSettingsStore().network.connectionType === NetworkConnectionType.Static"
+          v-show="!useSettingsStore().network.networkingDisabled"
-          v-model="useSettingsStore().network.staticIp"
+          v-if="tempSettingsStruct.connectionType === NetworkConnectionType.Static"
+          v-model="tempSettingsStruct.staticIp"
           :input-cols="12 - 4"
           label="Static IP"
           :rules="[(v) => isValidIPv4(v) || 'Invalid IPv4 address']"
-          :disabled="!(useSettingsStore().network.shouldManage && useSettingsStore().network.canManage)"
+          :disabled="
+            !tempSettingsStruct.shouldManage ||
+            !useSettingsStore().network.canManage ||
+            useSettingsStore().network.networkingDisabled
+          "
         />
         <pv-input
-          v-model="useSettingsStore().network.hostname"
+          v-show="!useSettingsStore().network.networkingDisabled"
+          v-model="tempSettingsStruct.hostname"
           label="Hostname"
           :input-cols="12 - 4"
           :rules="[(v) => isValidHostname(v) || 'Invalid hostname']"
-          :disabled="!(useSettingsStore().network.shouldManage && useSettingsStore().network.canManage)"
+          :disabled="
+            !tempSettingsStruct.shouldManage ||
+            !useSettingsStore().network.canManage ||
+            useSettingsStore().network.networkingDisabled
+          "
         />
         <v-divider class="pb-3" />
         <span style="font-weight: 700">Advanced Networking</span>
         <pv-switch
-          v-model="useSettingsStore().network.shouldManage"
+          v-show="!useSettingsStore().network.networkingDisabled"
-          :disabled="!useSettingsStore().network.canManage"
+          v-model="tempSettingsStruct.shouldManage"
+          :disabled="!useSettingsStore().network.canManage || useSettingsStore().network.networkingDisabled"
           label="Manage Device Networking"
           tooltip="If enabled, Photon will manage device hostname and network settings."
           :label-cols="4"
           class="pt-2"
         />
         <pv-select
+          v-show="!useSettingsStore().network.networkingDisabled"
           v-model="currentNetworkInterfaceIndex"
           label="NetworkManager interface"
-          :disabled="!(useSettingsStore().network.shouldManage && useSettingsStore().network.canManage)"
+          :disabled="
+            !tempSettingsStruct.shouldManage ||
+            !useSettingsStore().network.canManage ||
+            useSettingsStore().network.networkingDisabled
+          "
           :select-cols="12 - 4"
           tooltip="Name of the interface PhotonVision should manage the IP address of"
           :items="useSettingsStore().networkInterfaceNames"
@@ -158,8 +227,9 @@ const currentNetworkInterfaceIndex = computed<number>({
         <v-banner
           v-show="
             !useSettingsStore().networkInterfaceNames.length &&
-            useSettingsStore().network.shouldManage &&
+            tempSettingsStruct.shouldManage &&
-            useSettingsStore().network.canManage
+            useSettingsStore().network.canManage &&
+            !useSettingsStore().network.networkingDisabled
           "
           rounded
           color="red"
@@ -169,14 +239,14 @@ const currentNetworkInterfaceIndex = computed<number>({
           Photon cannot detect any wired connections! Please send program logs to the developers for help.
         </v-banner>
         <pv-switch
-          v-model="useSettingsStore().network.runNTServer"
+          v-model="tempSettingsStruct.runNTServer"
           label="Run NetworkTables Server (Debugging Only)"
           tooltip="If enabled, this device will create a NT server. This is useful for home debugging, but should be disabled on-robot."
-          class="mt-3 mb-3"
+          class="mt-3 mb-2"
           :label-cols="4"
         />
         <v-banner
-          v-show="useSettingsStore().network.runNTServer"
+          v-show="tempSettingsStruct.runNTServer"
           rounded
           color="red"
           text-color="white"
@@ -184,12 +254,29 @@ const currentNetworkInterfaceIndex = computed<number>({
         >
           This mode is intended for debugging; it should be off for proper usage. PhotonLib will NOT work!
         </v-banner>
+        <pv-switch
+          v-model="tempSettingsStruct.shouldPublishProto"
+          label="Also Publish Protobuf"
+          tooltip="If enabled, Photon will publish all pipeline results in both the Packet and Protobuf formats. This is useful for visualizing pipeline results from NT viewers such as glass and logging software such as AdvantageScope. Note: photon-lib will ignore this value and is not recommended on the field for performance."
+          class="mt-3 mb-2"
+          :label-cols="4"
+        />
+        <v-banner
+          v-show="tempSettingsStruct.shouldPublishProto"
+          rounded
+          color="red"
+          class="mb-3"
+          text-color="white"
+          icon="mdi-information-outline"
+        >
+          This mode is intended for debugging; it should be off for field use. You may notice a performance hit by using
+          this mode.
+        </v-banner>
       </v-form>
       <v-btn
         color="accent"
-        :class="useSettingsStore().network.runNTServer ? 'mt-3' : ''"
         style="color: black; width: 100%"
-        :disabled="!settingsValid && !useSettingsStore().network.runNTServer"
+        :disabled="!settingsValid || !settingsHaveChanged()"
         @click="saveGeneralSettings"
       >
         Save

photon-client/src/lib/MathUtils.ts

@@ -0,0 +1,13 @@
+export const mean = (values: number[]): number | undefined => {
+  if (values.length === 0) return undefined;
+  return values.reduce((acc, num) => acc + num, 0) / values.length;
+};
+
+export const angleModulus = (valueRad: number): number => {
+  while (valueRad < -Math.PI) valueRad += Math.PI * 2;
+  while (valueRad > Math.PI) valueRad -= Math.PI * 2;
+  return valueRad;
+};
+
+export const toDeg = (val: number) => val * (180.0 / Math.PI);
+export const toRad = (val: number) => val * (Math.PI / 180.0);

photon-client/src/lib/PhotonUtils.ts

@@ -0,0 +1,20 @@
+import type { Resolution } from "@/types/SettingTypes";
+
+export const resolutionsAreEqual = (a: Resolution, b: Resolution) => {
+  return a.height === b.height && a.width === b.width;
+};
+
+export const getResolutionString = (resolution: Resolution): string => `${resolution.width}x${resolution.height}`;
+
+export const parseJsonFile = async <T extends Record<string, any>>(file: File): Promise<T> => {
+  return new Promise((resolve, reject) => {
+    const fileReader = new FileReader();
+    fileReader.onload = (event) => {
+      const target: FileReader | null = event.target;
+      if (target === null) reject();
+      else resolve(JSON.parse(target.result as string) as T);
+    };
+    fileReader.onerror = (error) => reject(error);
+    fileReader.readAsText(file);
+  });
+};

photon-client/src/stores/StateStore.ts

@@ -1,7 +1,7 @@
 import { defineStore } from "pinia";
 import type { LogMessage } from "@/types/SettingTypes";
 import type { AutoReconnectingWebsocket } from "@/lib/AutoReconnectingWebsocket";
-import type { PipelineResult } from "@/types/PhotonTrackingTypes";
+import type { MultitagResult, PipelineResult } from "@/types/PhotonTrackingTypes";
 import type {
   WebsocketCalibrationData,
   WebsocketLogMessage,
@@ -25,6 +25,7 @@ interface StateStore {
   currentCameraIndex: number;
 
   backendResults: Record<string, PipelineResult>;
+  multitagResultBuffer: Record<string, MultitagResult[]>;
 
   colorPickingMode: boolean;
 
@@ -59,6 +60,7 @@ export const useStateStore = defineStore("state", {
       currentCameraIndex: 0,
 
       backendResults: {},
+      multitagResultBuffer: {},
 
       colorPickingMode: false,
 
@@ -80,6 +82,9 @@ export const useStateStore = defineStore("state", {
   getters: {
     currentPipelineResults(): PipelineResult | undefined {
       return this.backendResults[this.currentCameraIndex.toString()];
+    },
+    currentMultitagBuffer(): MultitagResult[] | undefined {
+      return this.multitagResultBuffer[this.currentCameraIndex.toString()];
     }
   },
   actions: {
@@ -105,6 +110,21 @@ export const useStateStore = defineStore("state", {
         ...this.backendResults,
         ...data
       };
+
+      for (const key in data) {
+        const multitagRes = data[key].multitagResult;
+
+        if (multitagRes) {
+          if (!this.multitagResultBuffer[key]) {
+            this.multitagResultBuffer[key] = [];
+          }
+
+          this.multitagResultBuffer[key].push(multitagRes);
+          if (this.multitagResultBuffer[key].length > 100) {
+            this.multitagResultBuffer[key].shift();
+          }
+        }
+      }
     },
     updateCalibrationStateValuesFromWebsocket(data: WebsocketCalibrationData) {
       this.calibrationData = {

photon-client/src/stores/settings/CameraSettingsStore.ts

@@ -3,7 +3,8 @@ import type {
   CalibrationBoardTypes,
   CameraCalibrationResult,
   CameraSettings,
-  ConfigurableCameraSettings,
+  CameraSettingsChangeRequest,
+  Resolution,
   RobotOffsetType,
   VideoFormat
 } from "@/types/SettingTypes";
@@ -13,6 +14,7 @@ import type { WebsocketCameraSettingsUpdate } from "@/types/WebsocketDataTypes";
 import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
 import type { ActiveConfigurablePipelineSettings, ActivePipelineSettings, PipelineType } from "@/types/PipelineTypes";
 import axios from "axios";
+import { resolutionsAreEqual } from "@/lib/PhotonUtils";
 
 interface CameraSettingsStore {
   cameras: CameraSettings[];
@@ -41,29 +43,37 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
       return this.currentCameraSettings.validVideoFormats[this.currentPipelineSettings.cameraVideoModeIndex];
     },
     isCurrentVideoFormatCalibrated(): boolean {
-      return this.currentCameraSettings.completeCalibrations.some(
+      return this.currentCameraSettings.completeCalibrations.some((v) =>
-        (v) =>
+        resolutionsAreEqual(v.resolution, this.currentVideoFormat.resolution)
-          v.resolution.width === this.currentVideoFormat.resolution.width &&
-          v.resolution.height === this.currentVideoFormat.resolution.height
       );
     },
     cameraNames(): string[] {
       return this.cameras.map((c) => c.nickname);
     },
+    currentCameraName(): string {
+      return this.cameraNames[useStateStore().currentCameraIndex];
+    },
     pipelineNames(): string[] {
       return this.currentCameraSettings.pipelineNicknames;
     },
+    currentPipelineName(): string {
+      return this.pipelineNames[useStateStore().currentCameraIndex];
+    },
     isDriverMode(): boolean {
       return this.currentCameraSettings.currentPipelineIndex === WebsocketPipelineType.DriverMode;
     },
     isCalibrationMode(): boolean {
       return this.currentCameraSettings.currentPipelineIndex == WebsocketPipelineType.Calib3d;
+    },
+    isCSICamera(): boolean {
+      return this.currentCameraSettings.isCSICamera;
     }
   },
   actions: {
     updateCameraSettingsFromWebsocket(data: WebsocketCameraSettingsUpdate[]) {
       this.cameras = data.map<CameraSettings>((d) => ({
         nickname: d.nickname,
+        uniqueName: d.uniqueName,
         fov: {
           value: d.fov,
           managedByVendor: !d.isFovConfigurable
@@ -89,33 +99,21 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
             standardDeviation: v.standardDeviation,
             mean: v.mean
           })),
-        completeCalibrations: d.calibrations.map<CameraCalibrationResult>((calib) => ({
+        completeCalibrations: d.calibrations,
-          resolution: {
+        isCSICamera: d.isCSICamera,
-            height: calib.height,
-            width: calib.width
-          },
-          distCoeffs: calib.distCoeffs,
-          standardDeviation: calib.standardDeviation,
-          perViewErrors: calib.perViewErrors,
-          intrinsics: calib.intrinsics
-        })),
         pipelineNicknames: d.pipelineNicknames,
         currentPipelineIndex: d.currentPipelineIndex,
-        pipelineSettings: d.currentPipelineSettings
+        pipelineSettings: d.currentPipelineSettings,
+        cameraQuirks: d.cameraQuirks
       }));
     },
     /**
      * Update the configurable camera settings.
      *
      * @param data camera settings to save.
-     * @param updateStore whether or not to update the store. This is useful if the input field already models the store reference.
      * @param cameraIndex the index of the camera.
      */
-    updateCameraSettings(
+    updateCameraSettings(data: CameraSettingsChangeRequest, cameraIndex: number = useStateStore().currentCameraIndex) {
-      data: ConfigurableCameraSettings,
-      updateStore = true,
-      cameraIndex: number = useStateStore().currentCameraIndex
-    ) {
       // The camera settings endpoint doesn't actually require all data, instead, it needs key data such as the FOV
       const payload = {
         settings: {
@@ -123,9 +121,6 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
         },
         index: cameraIndex
       };
-      if (updateStore) {
-        this.currentCameraSettings.fov.value = data.fov;
-      }
       return axios.post("/settings/camera", payload);
     },
     /**
@@ -315,6 +310,7 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
         patternWidth: number;
         patternHeight: number;
         boardType: CalibrationBoardTypes;
+        useMrCal: boolean;
       },
       cameraIndex: number = useStateStore().currentCameraIndex
     ) {
@@ -356,6 +352,16 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
       };
       return axios.post("/calibration/importFromCalibDB", payload, { headers: { "Content-Type": "text/plain" } });
     },
+    importCalibrationFromData(
+      data: { calibration: CameraCalibrationResult },
+      cameraIndex: number = useStateStore().currentCameraIndex
+    ) {
+      const payload = {
+        ...data,
+        cameraIndex: cameraIndex
+      };
+      return axios.post("/calibration/importFromData", payload);
+    },
     /**
      * Take a snapshot for the calibration processes
      *
@@ -404,6 +410,12 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
         cameraIndex: cameraIndex
       };
       useStateStore().websocket?.send(payload, true);
+    },
+    getCalibrationCoeffs(
+      resolution: Resolution,
+      cameraIndex: number = useStateStore().currentCameraIndex
+    ): CameraCalibrationResult | undefined {
+      return this.cameras[cameraIndex].completeCalibrations.find((v) => resolutionsAreEqual(v.resolution, resolution));
     }
   }
 });

photon-client/src/stores/settings/GeneralSettingsStore.ts

@@ -26,7 +26,9 @@ export const useSettingsStore = defineStore("settings", {
       version: undefined,
       gpuAcceleration: undefined,
       hardwareModel: undefined,
-      hardwarePlatform: undefined
+      hardwarePlatform: undefined,
+      mrCalWorking: true,
+      rknnSupported: false
     },
     network: {
       ntServerAddress: "",
@@ -36,6 +38,7 @@ export const useSettingsStore = defineStore("settings", {
       staticIp: "",
       hostname: "photonvision",
       runNTServer: false,
+      shouldPublishProto: false,
       networkInterfaceNames: [
         {
           connName: "Example Wired Connection",
@@ -96,24 +99,15 @@ export const useSettingsStore = defineStore("settings", {
         version: data.general.version || undefined,
         hardwareModel: data.general.hardwareModel || undefined,
         hardwarePlatform: data.general.hardwarePlatform || undefined,
-        gpuAcceleration: data.general.gpuAcceleration || undefined
+        gpuAcceleration: data.general.gpuAcceleration || undefined,
+        mrCalWorking: data.general.mrCalWorking,
+        rknnSupported: data.general.rknnSupported
       };
       this.lighting = data.lighting;
       this.network = data.networkSettings;
       this.currentFieldLayout = data.atfl;
     },
-    saveGeneralSettings() {
+    updateGeneralSettings(payload: Required<ConfigurableNetworkSettings>) {
-      const payload: Required<ConfigurableNetworkSettings> = {
-        connectionType: this.network.connectionType,
-        hostname: this.network.hostname,
-        networkManagerIface: this.network.networkManagerIface || "",
-        ntServerAddress: this.network.ntServerAddress,
-        runNTServer: this.network.runNTServer,
-        setDHCPcommand: this.network.setDHCPcommand || "",
-        setStaticCommand: this.network.setStaticCommand || "",
-        shouldManage: this.network.shouldManage,
-        staticIp: this.network.staticIp
-      };
       return axios.post("/settings/general", payload);
     },
     /**

photon-client/src/types/PhotonTrackingTypes.ts

@@ -1,3 +1,26 @@
+export interface Quaternion {
+  X: number;
+  Y: number;
+  Z: number;
+  W: number;
+}
+
+export interface Translation3d {
+  x: number;
+  y: number;
+  z: number;
+}
+
+export interface Rotation3d {
+  quaternion: Quaternion;
+}
+
+export interface Pose3d {
+  translation: Translation3d;
+  rotation: Rotation3d;
+}
+
+// TODO update backend to serialize this using correct layout
 export interface Transform3d {
   x: number;
   y: number;
@@ -6,16 +29,11 @@ export interface Transform3d {
   qx: number;
   qy: number;
   qz: number;
+  angle_x: number;
+  angle_y: number;
   angle_z: number;
 }
 
-export interface Quaternion {
-  X: number;
-  Y: number;
-  Z: number;
-  W: number;
-}
-
 export interface AprilTagFieldLayout {
   field: {
     length: number;
@@ -23,16 +41,7 @@ export interface AprilTagFieldLayout {
   };
   tags: {
     ID: number;
-    pose: {
+    pose: Pose3d;
-      translation: {
-        x: number;
-        y: number;
-        z: number;
-      };
-      rotation: {
-        quaternion: Quaternion;
-      };
-    };
   }[];
 }
 
@@ -45,6 +54,8 @@ export interface PhotonTarget {
   ambiguity: number;
   // -1 if not set
   fiducialId: number;
+  confidence: number;
+  classId: number;
   // undefined if 3d isn't enabled
   pose?: Transform3d;
 }
@@ -61,4 +72,6 @@ export interface PipelineResult {
   targets: PhotonTarget[];
   // undefined if multitag failed or non-tag pipeline
   multitagResult?: MultitagResult;
+  // Object detection class names -- empty if not doing object detection
+  classNames: string[];
 }

photon-client/src/types/PipelineTypes.ts

@@ -5,7 +5,8 @@ export enum PipelineType {
   Reflective = 2,
   ColoredShape = 3,
   AprilTag = 4,
-  Aruco = 5
+  Aruco = 5,
+  ObjectDetection = 6
 }
 
 export enum AprilTagFamily {
@@ -93,7 +94,11 @@ export type ConfigurablePipelineSettings = Partial<
     | "cornerDetectionStrategy"
   >
 >;
-export const DefaultPipelineSettings: PipelineSettings = {
+// Omitted settings are changed for all pipeline types
+export const DefaultPipelineSettings: Omit<
+  PipelineSettings,
+  "cameraGain" | "targetModel" | "ledMode" | "outputShowMultipleTargets" | "cameraExposure" | "pipelineType"
+> = {
   offsetRobotOffsetMode: RobotOffsetPointMode.None,
   streamingFrameDivisor: 0,
   offsetDualPointBArea: 0,
@@ -130,15 +135,7 @@ export const DefaultPipelineSettings: PipelineSettings = {
   cornerDetectionStrategy: 0,
   cornerDetectionAccuracyPercentage: 10,
   hsvSaturation: { first: 50, second: 255 },
-  contourIntersection: 1,
+  contourIntersection: 1
-
-  // These settings will be overridden by different pipeline types
-  cameraGain: -1,
-  targetModel: -1,
-  ledMode: false,
-  outputShowMultipleTargets: false,
-  cameraExposure: -1,
-  pipelineType: -1
 };
 
 export interface ReflectivePipelineSettings extends PipelineSettings {
@@ -264,6 +261,7 @@ export type ConfigurableArucoPipelineSettings = Partial<Omit<ArucoPipelineSettin
   ConfigurablePipelineSettings;
 export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
   ...DefaultPipelineSettings,
+  cameraGain: 75,
   outputShowMultipleTargets: true,
   targetModel: TargetModel.AprilTag6in_16h5,
   cameraExposure: -1,
@@ -284,14 +282,39 @@ export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
   doSingleTargetAlways: false
 };
 
+export interface ObjectDetectionPipelineSettings extends PipelineSettings {
+  pipelineType: PipelineType.ObjectDetection;
+  confidence: number;
+  nms: number;
+  box_thresh: number;
+}
+export type ConfigurableObjectDetectionPipelineSettings = Partial<
+  Omit<ObjectDetectionPipelineSettings, "pipelineType">
+> &
+  ConfigurablePipelineSettings;
+export const DefaultObjectDetectionPipelineSettings: ObjectDetectionPipelineSettings = {
+  ...DefaultPipelineSettings,
+  pipelineType: PipelineType.ObjectDetection,
+  cameraGain: 20,
+  targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
+  ledMode: true,
+  outputShowMultipleTargets: false,
+  cameraExposure: 6,
+  confidence: 0.9,
+  nms: 0.45,
+  box_thresh: 0.25
+};
+
 export type ActivePipelineSettings =
   | ReflectivePipelineSettings
   | ColoredShapePipelineSettings
   | AprilTagPipelineSettings
-  | ArucoPipelineSettings;
+  | ArucoPipelineSettings
+  | ObjectDetectionPipelineSettings;
 
 export type ActiveConfigurablePipelineSettings =
   | ConfigurableReflectivePipelineSettings
   | ConfigurableColoredShapePipelineSettings
   | ConfigurableAprilTagPipelineSettings
-  | ConfigurableArucoPipelineSettings;
+  | ConfigurableArucoPipelineSettings
+  | ConfigurableObjectDetectionPipelineSettings;

photon-client/src/types/SettingTypes.ts

@@ -1,10 +1,13 @@
 import { type ActivePipelineSettings, DefaultAprilTagPipelineSettings } from "@/types/PipelineTypes";
+import type { Pose3d } from "@/types/PhotonTrackingTypes";
 
 export interface GeneralSettings {
   version?: string;
   gpuAcceleration?: string;
   hardwareModel?: string;
   hardwarePlatform?: string;
+  mrCalWorking: boolean;
+  rknnSupported: boolean;
 }
 
 export interface MetricData {
@@ -36,14 +39,19 @@ export interface NetworkSettings {
   hostname: string;
   runNTServer: boolean;
   shouldManage: boolean;
+  shouldPublishProto: boolean;
   canManage: boolean;
   networkManagerIface?: string;
   setStaticCommand?: string;
   setDHCPcommand?: string;
   networkInterfaceNames: NetworkInterfaceType[];
+  networkingDisabled: boolean;
 }
 
-export type ConfigurableNetworkSettings = Omit<NetworkSettings, "canManage" | "networkInterfaceNames">;
+export type ConfigurableNetworkSettings = Omit<
+  NetworkSettings,
+  "canManage" | "networkInterfaceNames" | "networkingDisabled"
+>;
 
 export interface LightingSettings {
   supported: boolean;
@@ -76,24 +84,86 @@ export interface VideoFormat {
   diagonalFOV?: number;
   horizontalFOV?: number;
   verticalFOV?: number;
-  standardDeviation?: number;
   mean?: number;
 }
 
+export enum CvType {
+  CV_8U = 0,
+  CV_8S = 1,
+  CV_16U = 2,
+  CV_16S = 3,
+  CV_32S = 4,
+  CV_32F = 5,
+  CV_64F = 6,
+  CV_16F = 7
+}
+
+export interface JsonMatOfDouble {
+  rows: number;
+  cols: number;
+  type: CvType;
+  data: number[];
+}
+
+export interface JsonImageMat {
+  rows: number;
+  cols: number;
+  type: CvType;
+  data: string; // base64 encoded
+}
+
+export interface CvPoint3 {
+  x: number;
+  y: number;
+  z: number;
+}
+export interface CvPoint {
+  x: number;
+  y: number;
+}
+
+export interface BoardObservation {
+  locationInObjectSpace: CvPoint3[];
+  locationInImageSpace: CvPoint[];
+  reprojectionErrors: CvPoint[];
+  optimisedCameraToObject: Pose3d;
+  includeObservationInCalibration: boolean;
+  snapshotName: string;
+  snapshotData: JsonImageMat;
+}
+
 export interface CameraCalibrationResult {
   resolution: Resolution;
-  distCoeffs: number[];
+  cameraIntrinsics: JsonMatOfDouble;
-  standardDeviation: number;
+  distCoeffs: JsonMatOfDouble;
-  perViewErrors: number[];
+  observations: BoardObservation[];
-  intrinsics: number[];
+  calobjectWarp?: number[];
 }
 
-export interface ConfigurableCameraSettings {
+export enum ValidQuirks {
-  fov: number;
+  AWBGain = "AWBGain",
+  AdjustableFocus = "AdjustableFocus",
+  ArduOV9281 = "ArduOV9281",
+  ArduOV2311 = "ArduOV2311",
+  ArduCamCamera = "ArduCamCamera",
+  CompletelyBroken = "CompletelyBroken",
+  FPSCap100 = "FPSCap100",
+  Gain = "Gain",
+  PiCam = "PiCam",
+  StickyFPS = "StickyFPS"
+}
+
+export interface QuirkyCamera {
+  baseName: string;
+  usbVid: number;
+  usbPid: number;
+  displayName: string;
+  quirks: Record<ValidQuirks, boolean>;
 }
 
 export interface CameraSettings {
   nickname: string;
+  uniqueName: string;
 
   fov: {
     value: number;
@@ -111,13 +181,22 @@ export interface CameraSettings {
   currentPipelineIndex: number;
   pipelineNicknames: string[];
   pipelineSettings: ActivePipelineSettings;
+
+  cameraQuirks: QuirkyCamera;
+  isCSICamera: boolean;
+}
+
+export interface CameraSettingsChangeRequest {
+  fov: number;
+  quirksToChange: Record<ValidQuirks, boolean>;
 }
 
 export const PlaceholderCameraSettings: CameraSettings = {
   nickname: "Placeholder Camera",
+  uniqueName: "Placeholder Name",
   fov: {
     value: 70,
-    managedByVendor: true
+    managedByVendor: false
   },
   stream: {
     inputPort: 0,
@@ -140,11 +219,68 @@ export const PlaceholderCameraSettings: CameraSettings = {
       pixelFormat: "RGB"
     }
   ],
-  completeCalibrations: [],
+  completeCalibrations: [
+    {
+      resolution: { width: 1920, height: 1080 },
+      cameraIntrinsics: {
+        rows: 1,
+        cols: 1,
+        type: 1,
+        data: [1, 2, 3, 4, 5, 6, 7, 8, 9]
+      },
+      distCoeffs: {
+        rows: 1,
+        cols: 1,
+        type: 1,
+        data: [10, 11, 12, 13]
+      },
+      observations: [
+        {
+          locationInImageSpace: [
+            { x: 100, y: 100 },
+            { x: 210, y: 100 },
+            { x: 320, y: 101 }
+          ],
+          locationInObjectSpace: [{ x: 0, y: 0, z: 0 }],
+          optimisedCameraToObject: {
+            translation: { x: 1, y: 2, z: 3 },
+            rotation: { quaternion: { W: 1, X: 0, Y: 0, Z: 0 } }
+          },
+          reprojectionErrors: [
+            { x: 1, y: 1 },
+            { x: 2, y: 1 },
+            { x: 3, y: 1 }
+          ],
+          includeObservationInCalibration: false,
+          snapshotName: "img0.png",
+          snapshotData: { rows: 480, cols: 640, type: CvType.CV_8U, data: "" }
+        }
+      ]
+    }
+  ],
   pipelineNicknames: ["Placeholder Pipeline"],
   lastPipelineIndex: 0,
   currentPipelineIndex: 0,
-  pipelineSettings: DefaultAprilTagPipelineSettings
+  pipelineSettings: DefaultAprilTagPipelineSettings,
+  cameraQuirks: {
+    displayName: "Blank 1",
+    baseName: "Blank 2",
+    usbVid: -1,
+    usbPid: -1,
+    quirks: {
+      AWBGain: false,
+      AdjustableFocus: false,
+      ArduOV9281: false,
+      ArduOV2311: false,
+      ArduCamCamera: false,
+      CompletelyBroken: false,
+      FPSCap100: false,
+      Gain: false,
+      PiCam: false,
+      StickyFPS: false
+    }
+  },
+  isCSICamera: false
 };
 
 export enum CalibrationBoardTypes {

photon-client/src/types/WebsocketDataTypes.ts

@@ -1,4 +1,12 @@
-import type { GeneralSettings, LightingSettings, LogLevel, MetricData, NetworkSettings } from "@/types/SettingTypes";
+import type {
+  CameraCalibrationResult,
+  GeneralSettings,
+  LightingSettings,
+  LogLevel,
+  MetricData,
+  NetworkSettings,
+  QuirkyCamera
+} from "@/types/SettingTypes";
 import type { ActivePipelineSettings } from "@/types/PipelineTypes";
 import type { AprilTagFieldLayout, PipelineResult } from "@/types/PhotonTrackingTypes";
 
@@ -20,15 +28,6 @@ export interface WebsocketNumberPair {
   second: number;
 }
 
-export interface WebsocketCompleteCalib {
-  distCoeffs: number[];
-  height: number;
-  width: number;
-  standardDeviation: number;
-  perViewErrors: number[];
-  intrinsics: number[];
-}
-
 export type WebsocketVideoFormat = Record<
   number,
   {
@@ -46,16 +45,19 @@ export type WebsocketVideoFormat = Record<
 >;
 
 export interface WebsocketCameraSettingsUpdate {
-  calibrations: WebsocketCompleteCalib[];
+  calibrations: CameraCalibrationResult[];
   currentPipelineIndex: number;
   currentPipelineSettings: ActivePipelineSettings;
   fov: number;
   inputStreamPort: number;
   isFovConfigurable: boolean;
+  isCSICamera: boolean;
   nickname: string;
+  uniqueName: string;
   outputStreamPort: number;
   pipelineNicknames: string[];
   videoFormatList: WebsocketVideoFormat;
+  cameraQuirks: QuirkyCamera;
 }
 export interface WebsocketNTUpdate {
   connected: boolean;
@@ -99,5 +101,6 @@ export enum WebsocketPipelineType {
   Reflective = 0,
   ColoredShape = 1,
   AprilTag = 2,
-  Aruco = 3
+  Aruco = 3,
+  ObjectDetection = 4
 }

photon-client/src/views/CameraSettingsView.vue

@@ -5,6 +5,7 @@ import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { computed } from "vue";
 import CamerasView from "@/components/cameras/CamerasView.vue";
 import { useStateStore } from "@/stores/StateStore";
+import CameraControlCard from "@/components/cameras/CameraControlCard.vue";
 
 const cameraViewType = computed<number[]>({
   get: (): number[] => {
@@ -40,6 +41,7 @@ const cameraViewType = computed<number[]>({
       <v-col cols="12" md="7">
         <CamerasCard />
         <CalibrationCard />
+        <CameraControlCard />
       </v-col>
       <v-col class="pl-md-3 pt-3 pt-md-0" cols="12" md="5">
         <CamerasView v-model="cameraViewType" />

photon-client/tsconfig.config.json

@@ -1,6 +1,6 @@
 {
-  "extends": "@vue/tsconfig/tsconfig.node.json",
+  "extends": "@vue/tsconfig/tsconfig.json",
-  "include": ["vite.config.*", "vitest.config.*", "cypress.config.*"],
+  "include": ["vite.config.*"],
   "compilerOptions": {
     "composite": true,
     "types": ["node"]

photon-client/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "extends": "@vue/tsconfig/tsconfig.web.json",
+  "extends": "@vue/tsconfig/tsconfig.dom.json",
   "include": ["env.d.ts", "src/**/*", "src/**/*.vue"],
   "compilerOptions": {
     "moduleResolution": "node",
@@ -7,6 +7,7 @@
     "strict": true,
     "removeComments": true,
     "sourceMap": true,
+    "module": "ESNext",
     "types": ["node"],
     "baseUrl": ".",
     "paths": {

photon-client/vite.config.ts

@@ -10,25 +10,22 @@ export default defineConfig({
   plugins: [
     Vue2(),
     Components({
-      resolvers: [
+      resolvers: [VuetifyResolver()],
-          VuetifyResolver()
-      ],
       dts: true,
       transformer: "vue2",
-      types: [{
+      types: [
-        from: "vue-router",
+        {
-        names: ["RouterLink", "RouterView"]
+          from: "vue-router",
-      }],
+          names: ["RouterLink", "RouterView"]
+        }
+      ],
       version: 2.7
     })
   ],
   css: {
     preprocessorOptions: {
       sass: {
-        additionalData: [
+        additionalData: ["@import \"@/assets/styles/variables.scss\"", ""].join("\n")
-          "@import \"@/assets/styles/variables.scss\"",
-          ""
-        ].join("\n")
       }
     }
   },

photon-core/build.gradle

@@ -6,33 +6,61 @@ import java.nio.file.Path
 
 apply from: "${rootDir}/shared/common.gradle"
 
+wpilibTools.deps.wpilibVersion = wpi.versions.wpilibVersion.get()
+
+def nativeConfigName = 'wpilibNatives'
+def nativeConfig = configurations.create(nativeConfigName)
+
+def nativeTasks = wpilibTools.createExtractionTasks {
+    configurationName = nativeConfigName
+}
+
+nativeTasks.addToSourceSetResources(sourceSets.main)
+
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("wpimath")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("wpinet")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("wpiutil")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("ntcore")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("cscore")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("apriltag")
+nativeConfig.dependencies.add wpilibTools.deps.wpilib("hal")
+nativeConfig.dependencies.add wpilibTools.deps.wpilibOpenCv("frc" + wpi.frcYear.get(), wpi.versions.opencvVersion.get())
+
 dependencies {
-    implementation project(':photon-targeting')
-
-    implementation "io.javalin:javalin:$javalinVersion"
-
-    implementation 'org.msgpack:msgpack-core:0.9.0'
-    implementation 'org.msgpack:jackson-dataformat-msgpack:0.9.0'
-
     // JOGL stuff (currently we only distribute for aarch64, which is Pi 4)
     implementation "org.jogamp.gluegen:gluegen-rt:$joglVersion"
     implementation "org.jogamp.jogl:jogl-all:$joglVersion"
-
     implementation "org.jogamp.gluegen:gluegen-rt:$joglVersion:natives-linux-aarch64"
     implementation "org.jogamp.jogl:jogl-all:$joglVersion:natives-linux-aarch64"
 
     // Zip
     implementation 'org.zeroturnaround:zt-zip:1.14'
 
-    implementation wpilibTools.deps.wpilibJava("apriltag")
-
     implementation "org.xerial:sqlite-jdbc:3.41.0.0"
+    def rknnjniversion = "dev-v2024.0.0-44-g8022c40"
+    implementation "org.photonvision:rknn_jni-jni:$rknnjniversion:linuxarm64"
+    implementation "org.photonvision:rknn_jni-java:$rknnjniversion"
+    implementation "org.photonvision:photon-libcamera-gl-driver-jni:$photonGlDriverLibVersion:linuxarm64"
+    implementation "org.photonvision:photon-libcamera-gl-driver-java:$photonGlDriverLibVersion"
+
+    implementation "org.photonvision:photon-mrcal-java:$mrcalVersion"
+
+    // Only include mrcal natives on platforms that we build for
+    if (!(jniPlatform in [
+                "osxx86-64",
+                "osxarm64",
+                "linuxarm32"
+            ])) {
+        implementation "org.photonvision:photon-mrcal-jni:$mrcalVersion:$wpilibNativeName"
+    }
+
+    testImplementation group: 'org.junit-pioneer' , name: 'junit-pioneer', version: '2.2.0'
 }
 
-task writeCurrentVersionJava {
+task writeCurrentVersion {
     def versionFileIn = file("${rootDir}/shared/PhotonVersion.java.in")
     writePhotonVersionFile(versionFileIn, Path.of("$projectDir", "src", "main", "java", "org", "photonvision", "PhotonVersion.java"),
             versionString)
 }
 
-build.dependsOn writeCurrentVersionJava
+build.dependsOn writeCurrentVersion

photon-core/src/main/java/org/photonvision/common/ProgramStatus.java

@@ -20,6 +20,5 @@ package org.photonvision.common;
 public enum ProgramStatus {
     UHOH,
     RUNNING,
-    RUNNING_NT,
+    RUNNING_NT
-    RUNNING_NT_TARGET
 }

photon-core/src/main/java/org/photonvision/common/configuration/CameraConfiguration.java

@@ -27,6 +27,7 @@ import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
 import org.photonvision.vision.camera.CameraType;
+import org.photonvision.vision.camera.QuirkyCamera;
 import org.photonvision.vision.pipeline.CVPipelineSettings;
 import org.photonvision.vision.pipeline.DriverModePipelineSettings;
 import org.photonvision.vision.processes.PipelineManager;
@@ -46,6 +47,8 @@ public class CameraConfiguration {
     /** Can be either path (ex /dev/videoX) or index (ex 1). */
     public String path = "";
 
+    public QuirkyCamera cameraQuirks;
+
     @JsonIgnore public String[] otherPaths = {};
 
     public CameraType cameraType = CameraType.UsbCamera;
@@ -93,6 +96,7 @@ public class CameraConfiguration {
             @JsonProperty("FOV") double FOV,
             @JsonProperty("path") String path,
             @JsonProperty("cameraType") CameraType cameraType,
+            @JsonProperty("cameraQuirks") QuirkyCamera cameraQuirks,
             @JsonProperty("calibration") List<CameraCalibrationCoefficients> calibrations,
             @JsonProperty("currentPipelineIndex") int currentPipelineIndex) {
         this.baseName = baseName;
@@ -101,6 +105,7 @@ public class CameraConfiguration {
         this.FOV = FOV;
         this.path = path;
         this.cameraType = cameraType;
+        this.cameraQuirks = cameraQuirks;
         this.calibrations = calibrations != null ? calibrations : new ArrayList<>();
         this.currentPipelineIndex = currentPipelineIndex;
 
@@ -165,6 +170,8 @@ public class CameraConfiguration {
                 + Arrays.toString(otherPaths)
                 + ", cameraType="
                 + cameraType
+                + ", cameraQuirks="
+                + cameraQuirks
                 + ", FOV="
                 + FOV
                 + ", calibrations="

photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java

@@ -50,6 +50,10 @@ public class ConfigManager {
     private final Thread settingsSaveThread;
     private long saveRequestTimestamp = -1;
 
+    // special case flag to disable flushing settings to disk at shutdown. Avoids the jvm shutdown
+    // hook overwriting the settings we just uploaded
+    private boolean flushOnShutdown = true;
+
     enum ConfigSaveStrategy {
         SQL,
         LEGACY,
@@ -62,12 +66,13 @@ public class ConfigManager {
 
     public static ConfigManager getInstance() {
         if (INSTANCE == null) {
+            Path rootFolder = PathManager.getInstance().getRootFolder();
             switch (m_saveStrat) {
                 case SQL:
-                    INSTANCE = new ConfigManager(getRootFolder(), new SqlConfigProvider(getRootFolder()));
+                    INSTANCE = new ConfigManager(rootFolder, new SqlConfigProvider(rootFolder));
                     break;
                 case LEGACY:
-                    INSTANCE = new ConfigManager(getRootFolder(), new LegacyConfigProvider(getRootFolder()));
+                    INSTANCE = new ConfigManager(rootFolder, new LegacyConfigProvider(rootFolder));
                     break;
                 case ATOMIC_ZIP:
                     // not yet done, fall through
@@ -78,7 +83,7 @@ public class ConfigManager {
         return INSTANCE;
     }
 
-    private static final Logger logger = new Logger(ConfigManager.class, LogGroup.General);
+    private static final Logger logger = new Logger(ConfigManager.class, LogGroup.Config);
 
     private void translateLegacyIfPresent(Path folderPath) {
         if (!(m_provider instanceof SqlConfigProvider)) {
@@ -167,7 +172,7 @@ public class ConfigManager {
     }
 
     private static Path getRootFolder() {
-        return Path.of("photonvision_config");
+        return PathManager.getInstance().getRootFolder();
     }
 
     ConfigManager(Path configDirectory, ConfigProvider provider) {
@@ -295,4 +300,26 @@ public class ConfigManager {
             }
         }
     }
+
+    /** Get (and create if not present) the subfolder where ML models are stored */
+    public File getModelsDirectory() {
+        var ret = new File(configDirectoryFile, "models");
+        if (!ret.exists()) ret.mkdirs();
+        return ret;
+    }
+
+    /**
+     * Disable flushing settings to disk as part of our JVM exit hook. Used to prevent uploading all
+     * settings from getting its new configs overwritten at program exit and before theyre all loaded.
+     */
+    public void disableFlushOnShutdown() {
+        this.flushOnShutdown = false;
+    }
+
+    public void onJvmExit() {
+        if (flushOnShutdown) {
+            logger.info("Force-flushing settings...");
+            saveToDisk();
+        }
+    }
 }

photon-core/src/main/java/org/photonvision/common/configuration/DatabaseSchema.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.configuration;
+
+/**
+ * Add migrations by adding the SQL commands for each migration sequentially to this array. DO NOT
+ * edit or delete existing SQL commands. That will lead to producing an icompatible database.
+ *
+ * <p>You can use multiple SQL statements in one migration step as long as you separate them with a
+ * semicolon (;).
+ */
+public final class DatabaseSchema {
+    public static final String[] migrations = {
+        // #1 - initial schema
+        "CREATE TABLE IF NOT EXISTS global (\n"
+                + " filename TINYTEXT PRIMARY KEY,\n"
+                + " contents mediumtext NOT NULL\n"
+                + ");"
+                + "CREATE TABLE IF NOT EXISTS cameras (\n"
+                + " unique_name TINYTEXT PRIMARY KEY,\n"
+                + " config_json text NOT NULL,\n"
+                + " drivermode_json text NOT NULL,\n"
+                + " pipeline_jsons mediumtext NOT NULL\n"
+                + ");",
+        // #2 - add column otherpaths_json
+        "ALTER TABLE cameras ADD COLUMN otherpaths_json TEXT NOT NULL DEFAULT '[]';",
+        // add future migrations here
+    };
+
+    // Constants for the tables and column to help prevent typos in SQL queries
+    // Update these tables to keep them constant with the current schema
+    public final class Tables {
+        // These constants should match the current SQL name of each table
+        public static final String GLOBAL = "global";
+        public static final String CAMERAS = "cameras";
+    }
+
+    public final class Columns {
+        // These constants should match the current SQL name of each column
+        static final String GLB_FILENAME = "filename";
+        static final String GLB_CONTENTS = "contents";
+
+        static final String CAM_UNIQUE_NAME = "unique_name";
+        static final String CAM_CONFIG_JSON = "config_json";
+        static final String CAM_DRIVERMODE_JSON = "drivermode_json";
+        static final String CAM_PIPELINE_JSONS = "pipeline_jsons";
+        static final String CAM_OTHERPATHS_JSON = "otherpaths_json";
+    }
+}

photon-core/src/main/java/org/photonvision/common/configuration/LegacyConfigProvider.java

@@ -196,7 +196,7 @@ class LegacyConfigProvider extends ConfigProvider {
             }
         }
         if (atfl == null) {
-            logger.info("Loading default apriltags for 2023 field...");
+            logger.info("Loading default apriltags for 2024 field...");
             try {
                 atfl = AprilTagFields.kDefaultField.loadAprilTagLayoutField();
             } catch (UncheckedIOException e) {

photon-core/src/main/java/org/photonvision/common/configuration/NetworkConfig.java

@@ -37,6 +37,7 @@ public class NetworkConfig {
     public String hostname = "photonvision";
     public boolean runNTServer = false;
     public boolean shouldManage;
+    public boolean shouldPublishProto = false;
 
     @JsonIgnore public static final String NM_IFACE_STRING = "${interface}";
     @JsonIgnore public static final String NM_IP_STRING = "${ipaddr}";
@@ -72,6 +73,7 @@ public class NetworkConfig {
             @JsonProperty("hostname") String hostname,
             @JsonProperty("runNTServer") boolean runNTServer,
             @JsonProperty("shouldManage") boolean shouldManage,
+            @JsonProperty("shouldPublishProto") boolean shouldPublishProto,
             @JsonProperty("networkManagerIface") String networkManagerIface,
             @JsonProperty("setStaticCommand") String setStaticCommand,
             @JsonProperty("setDHCPcommand") String setDHCPcommand) {
@@ -80,6 +82,7 @@ public class NetworkConfig {
         this.staticIp = staticIp;
         this.hostname = hostname;
         this.runNTServer = runNTServer;
+        this.shouldPublishProto = shouldPublishProto;
         this.networkManagerIface = networkManagerIface;
         this.setStaticCommand = setStaticCommand;
         this.setDHCPcommand = setDHCPcommand;

photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.configuration;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+public class NeuralNetworkModelManager {
+    private static NeuralNetworkModelManager INSTANCE;
+    private static final Logger logger = new Logger(NeuralNetworkModelManager.class, LogGroup.Config);
+
+    private final String MODEL_NAME = "note-640-640-yolov5s.rknn";
+    private File defaultModelFile;
+    private List<String> labels;
+
+    public static NeuralNetworkModelManager getInstance() {
+        if (INSTANCE == null) {
+            INSTANCE = new NeuralNetworkModelManager();
+        }
+        return INSTANCE;
+    }
+
+    /**
+     * Perform initial setup and extract default model from JAR to the filesystem
+     *
+     * @param modelsFolder Where models live
+     */
+    public void initialize(File modelsFolder) {
+        var modelResourcePath = "/models/" + MODEL_NAME;
+        this.defaultModelFile = new File(modelsFolder, MODEL_NAME);
+        extractResource(modelResourcePath, defaultModelFile);
+
+        File labelsFile = new File(modelsFolder, "labels.txt");
+        var labelResourcePath = "/models/" + labelsFile.getName();
+        extractResource(labelResourcePath, labelsFile);
+
+        try {
+            labels = Files.readAllLines(Paths.get(labelsFile.getPath()));
+        } catch (IOException e) {
+            logger.error("Error reading labels.txt", e);
+        }
+    }
+
+    private void extractResource(String resourcePath, File outputFile) {
+        try (var in = NeuralNetworkModelManager.class.getResourceAsStream(resourcePath)) {
+            if (in == null) {
+                logger.error("Failed to find jar resource at " + resourcePath);
+                return;
+            }
+
+            if (!outputFile.exists()) {
+                try (FileOutputStream fos = new FileOutputStream(outputFile)) {
+                    int read = -1;
+                    byte[] buffer = new byte[1024];
+                    while ((read = in.read(buffer)) != -1) {
+                        fos.write(buffer, 0, read);
+                    }
+                } catch (IOException e) {
+                    logger.error("Error extracting resource to " + outputFile.toPath().toString(), e);
+                }
+            } else {
+                logger.info(
+                        "File " + outputFile.toPath().toString() + " already exists. Skipping extraction.");
+            }
+        } catch (IOException e) {
+            logger.error("Error finding jar resource " + resourcePath, e);
+        }
+    }
+
+    public File getDefaultRknnModel() {
+        return defaultModelFile;
+    }
+
+    public List<String> getLabels() {
+        return labels;
+    }
+}

photon-core/src/main/java/org/photonvision/common/configuration/PathManager.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.configuration;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
+import java.util.Date;
+
+public class PathManager {
+    private static PathManager INSTANCE;
+
+    final File configDirectoryFile;
+
+    public static PathManager getInstance() {
+        if (INSTANCE == null) {
+            INSTANCE = new PathManager();
+        }
+        return INSTANCE;
+    }
+
+    private PathManager() {
+        this.configDirectoryFile = new File(getRootFolder().toUri());
+    }
+
+    public Path getRootFolder() {
+        return Path.of("photonvision_config");
+    }
+
+    public Path getLogsDir() {
+        return Path.of(configDirectoryFile.toString(), "logs");
+    }
+
+    public static final String LOG_PREFIX = "photonvision-";
+    public static final String LOG_EXT = ".log";
+    public static final String LOG_DATE_TIME_FORMAT = "yyyy-M-d_hh-mm-ss";
+
+    public String taToLogFname(TemporalAccessor date) {
+        var dateString = DateTimeFormatter.ofPattern(LOG_DATE_TIME_FORMAT).format(date);
+        return LOG_PREFIX + dateString + LOG_EXT;
+    }
+
+    public Path getLogPath() {
+        var logFile = Path.of(this.getLogsDir().toString(), taToLogFname(LocalDateTime.now())).toFile();
+        if (!logFile.getParentFile().exists()) logFile.getParentFile().mkdirs();
+        return logFile.toPath();
+    }
+
+    public Date logFnameToDate(String fname) throws ParseException {
+        // Strip away known unneeded portions of the log file name
+        fname = fname.replace(LOG_PREFIX, "").replace(LOG_EXT, "");
+        DateFormat format = new SimpleDateFormat(LOG_DATE_TIME_FORMAT);
+        return format.parse(fname);
+    }
+}

photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java

@@ -25,9 +25,14 @@ import java.util.Map;
 import java.util.stream.Collectors;
 import org.photonvision.PhotonVersion;
 import org.photonvision.common.hardware.Platform;
+import org.photonvision.common.networking.NetworkManager;
 import org.photonvision.common.networking.NetworkUtils;
 import org.photonvision.common.util.SerializationUtils;
-import org.photonvision.raspi.LibCameraJNI;
+import org.photonvision.jni.RknnDetectorJNI;
+import org.photonvision.mrcal.MrCalJNILoader;
+import org.photonvision.raspi.LibCameraJNILoader;
+import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
+import org.photonvision.vision.camera.QuirkyCamera;
 import org.photonvision.vision.processes.VisionModule;
 import org.photonvision.vision.processes.VisionModuleManager;
 import org.photonvision.vision.processes.VisionSource;
@@ -117,6 +122,7 @@ public class PhotonConfiguration {
         // Hack active interfaces into networkSettings
         var netConfigMap = networkConfig.toHashMap();
         netConfigMap.put("networkInterfaceNames", NetworkUtils.getAllWiredInterfaces());
+        netConfigMap.put("networkingDisabled", NetworkManager.getInstance().networkingIsDisabled);
 
         settingsSubmap.put("networkSettings", netConfigMap);
 
@@ -136,9 +142,11 @@ public class PhotonConfiguration {
         generalSubmap.put("version", PhotonVersion.versionString);
         generalSubmap.put(
                 "gpuAcceleration",
-                LibCameraJNI.isSupported()
+                LibCameraJNILoader.isSupported()
-                        ? "Zerocopy Libcamera on " + LibCameraJNI.getSensorModel().getFriendlyName()
+                        ? "Zerocopy Libcamera Working"
                         : ""); // TODO add support for other types of GPU accel
+        generalSubmap.put("mrCalWorking", MrCalJNILoader.getInstance().isLoaded());
+        generalSubmap.put("rknnSupported", RknnDetectorJNI.getInstance().isLoaded());
         generalSubmap.put("hardwareModel", hardwareConfig.deviceName);
         generalSubmap.put("hardwarePlatform", Platform.getPlatformName());
         settingsSubmap.put("general", generalSubmap);
@@ -166,13 +174,16 @@ public class PhotonConfiguration {
         public double fov;
 
         public String nickname;
+        public String uniqueName;
         public HashMap<String, Object> currentPipelineSettings;
         public int currentPipelineIndex;
         public List<String> pipelineNicknames;
         public HashMap<Integer, HashMap<String, Object>> videoFormatList;
         public int outputStreamPort;
         public int inputStreamPort;
-        public List<HashMap<String, Object>> calibrations;
+        public List<CameraCalibrationCoefficients> calibrations;
         public boolean isFovConfigurable = true;
+        public QuirkyCamera cameraQuirks;
+        public boolean isCSICamera;
     }
 }

photon-core/src/main/java/org/photonvision/common/configuration/SqlConfigProvider.java

@@ -30,6 +30,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
+import org.photonvision.common.configuration.DatabaseSchema.Columns;
+import org.photonvision.common.configuration.DatabaseSchema.Tables;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.util.file.JacksonUtils;
@@ -45,14 +47,9 @@ import org.photonvision.vision.pipeline.DriverModePipelineSettings;
  * <p>Global has one row per global config file (like hardware settings and network settings)
  */
 public class SqlConfigProvider extends ConfigProvider {
-    private final Logger logger = new Logger(SqlConfigProvider.class, LogGroup.Config);
+    private static final Logger logger = new Logger(SqlConfigProvider.class, LogGroup.Config);
-
-    static class TableKeys {
-        static final String CAM_UNIQUE_NAME = "unique_name";
-        static final String CONFIG_JSON = "config_json";
-        static final String DRIVERMODE_JSON = "drivermode_json";
-        static final String PIPELINE_JSONS = "pipeline_jsons";
 
+    static class GlobalKeys {
         static final String NETWORK_CONFIG = "networkConfig";
         static final String HARDWARE_CONFIG = "hardwareConfig";
         static final String HARDWARE_SETTINGS = "hardwareSettings";
@@ -60,14 +57,24 @@ public class SqlConfigProvider extends ConfigProvider {
     }
 
     private static final String dbName = "photon.sqlite";
+    // private final File rootFolder;
     private final String dbPath;
+    private final String url;
 
     private final Object m_mutex = new Object();
-    private final File rootFolder;
 
-    public SqlConfigProvider(Path rootFolder) {
+    public SqlConfigProvider(Path rootPath) {
-        this.rootFolder = rootFolder.toFile();
+        File rootFolder = rootPath.toFile();
+        // Make sure root dir exists
+        if (!rootFolder.exists()) {
+            if (rootFolder.mkdirs()) {
+                logger.debug("Root config folder did not exist. Created!");
+            } else {
+                logger.error("Failed to create root config folder!");
+            }
+        }
         dbPath = Path.of(rootFolder.toString(), dbName).toAbsolutePath().toString();
+        url = "jdbc:sqlite:" + dbPath;
         logger.debug("Using database " + dbPath);
         initDatabase();
     }
@@ -79,90 +86,136 @@ public class SqlConfigProvider extends ConfigProvider {
         return config;
     }
 
-    private Connection createConn() {
+    private Connection createConn(boolean autoCommit) {
-        String url = "jdbc:sqlite:" + dbPath;
+        Connection conn = null;
-
         try {
-            var conn = DriverManager.getConnection(url);
+            conn = DriverManager.getConnection(url);
-            conn.setAutoCommit(false);
+            conn.setAutoCommit(autoCommit);
-            return conn;
         } catch (SQLException e) {
             logger.error("Error creating connection", e);
-            return null;
         }
+        return conn;
+    }
+
+    private Connection createConn() {
+        return createConn(false);
     }
 
     private void tryCommit(Connection conn) {
         try {
             conn.commit();
-        } catch (SQLException e) {
+        } catch (SQLException e1) {
-            logger.error("Err committing changes: ", e);
+            logger.error("Err committing changes: ", e1);
             try {
                 conn.rollback();
-            } catch (SQLException e1) {
+            } catch (SQLException e2) {
-                logger.error("Err rolling back changes: ", e);
+                logger.error("Err rolling back changes: ", e2);
             }
         }
     }
 
-    private void initDatabase() {
+    private int getIntPragma(String pragma) {
-        // Make sure root dir exists
+        int retval = 0;
+        try (Connection conn = createConn(true);
+                Statement stmt = conn.createStatement()) {
+            ResultSet rs = stmt.executeQuery("PRAGMA " + pragma + ";");
+            retval = rs.getInt(1);
+        } catch (SQLException e) {
+            logger.error("Error querying " + pragma, e);
+        }
+        return retval;
+    }
 
-        if (!rootFolder.exists()) {
+    private int getSchemaVersion() {
-            if (rootFolder.mkdirs()) {
+        return getIntPragma("schema_version");
-                logger.debug("Root config folder did not exist. Created!");
+    }
-            } else {
+
-                logger.error("Failed to create root config folder!");
+    public int getUserVersion() {
+        return getIntPragma("user_version");
+    }
+
+    private void setUserVersion(Connection conn, int value) {
+        try (Statement stmt = conn.createStatement()) {
+            stmt.execute("PRAGMA user_version = " + value + ";");
+        } catch (SQLException e) {
+            logger.error("Error setting user_version to ", e);
+        }
+    }
+
+    private void doMigration(int index) throws SQLException {
+        logger.debug("Running migration step " + index);
+        try (Connection conn = createConn();
+                Statement stmt = conn.createStatement()) {
+            for (String sql : DatabaseSchema.migrations[index].split(";")) {
+                stmt.addBatch(sql);
+            }
+            stmt.executeBatch();
+            setUserVersion(conn, index + 1);
+            tryCommit(conn);
+        } catch (SQLException e) {
+            logger.error("Error with migration step " + index, e);
+            throw e;
+        }
+    }
+
+    private void initDatabase() {
+        int userVersion = getUserVersion();
+        int expectedVersion = DatabaseSchema.migrations.length;
+
+        if (userVersion < expectedVersion) {
+            // older database, run migrations
+
+            // first, check to see if this is one of the ones from 2024 beta that need special handling
+            if (userVersion == 0 && getSchemaVersion() > 0) {
+                String sql =
+                        "SELECT COUNT(*) AS CNTREC FROM pragma_table_info('cameras') WHERE name='otherpaths_json';";
+                try (Connection conn = createConn(true);
+                        Statement stmt = conn.createStatement();
+                        ResultSet rs = stmt.executeQuery(sql); ) {
+                    if (rs.getInt("CNTREC") == 0) {
+                        // need to add otherpaths_json
+                        userVersion = 1;
+                    } else {
+                        // already there, no need to add the column
+                        userVersion = 2;
+                    }
+                    setUserVersion(conn, userVersion);
+                } catch (SQLException e) {
+                    logger.error(
+                            "Could not determine the version of the database. Try deleting "
+                                    + dbName
+                                    + "and restart photonvision.",
+                            e);
+                }
+            }
+
+            logger.debug("Older database version. Migrating ... ");
+            try {
+                for (int index = userVersion; index < expectedVersion; index++) {
+                    doMigration(index);
+                }
+                logger.debug("Database migration complete");
+            } catch (SQLException e) {
+                logger.error("Error with database migration", e);
             }
         }
 
-        Connection conn = null;
+        // Warn if the database still isn't at the correct version
-        Statement createGlobalTableStatement = null, createCameraTableStatement = null;
+        userVersion = getUserVersion();
-        try {
+        if (userVersion > expectedVersion) {
-            conn = createConn();
+            // database must be from a newer version, so warn
-            if (conn == null) {
+            logger.warn(
-                logger.error("No connection, cannot init db");
+                    "This database is from a newer version of PhotonVision. Check that you are running the right version of PhotonVision.");
-                return;
+        } else if (userVersion < expectedVersion) {
-            }
+            // migration didn't work, so warn
-
+            logger.warn(
-            // Create global settings table. Just a dumb table with list of jsons and their
+                    "This database migration failed. Expected version: "
-            // name
+                            + expectedVersion
-            try {
+                            + ", got version: "
-                createGlobalTableStatement = conn.createStatement();
+                            + userVersion);
-                String sql =
+        } else {
-                        "CREATE TABLE IF NOT EXISTS global (\n"
+            // migration worked
-                                + " filename TINYTEXT PRIMARY KEY,\n"
+            logger.info("Using correct database version: " + userVersion);
-                                + " contents mediumtext NOT NULL\n"
-                                + ");";
-                createGlobalTableStatement.execute(sql);
-            } catch (SQLException e) {
-                logger.error("Err creating global table", e);
-            }
-
-            // Create cameras table, key is the camera unique name
-            try {
-                createCameraTableStatement = conn.createStatement();
-                var sql =
-                        "CREATE TABLE IF NOT EXISTS cameras (\n"
-                                + " unique_name TINYTEXT PRIMARY KEY,\n"
-                                + " config_json text NOT NULL,\n"
-                                + " drivermode_json text NOT NULL,\n"
-                                + " pipeline_jsons mediumtext NOT NULL\n"
-                                + ");";
-                createCameraTableStatement.execute(sql);
-            } catch (SQLException e) {
-                logger.error("Err creating cameras table", e);
-            }
-
-            this.tryCommit(conn);
-        } finally {
-            try {
-                if (createGlobalTableStatement != null) createGlobalTableStatement.close();
-                if (createCameraTableStatement != null) createCameraTableStatement.close();
-                if (conn != null) conn.close();
-            } catch (SQLException e) {
-                e.printStackTrace();
-            }
         }
     }
 
@@ -210,7 +263,7 @@ public class SqlConfigProvider extends ConfigProvider {
             try {
                 hardwareConfig =
                         JacksonUtils.deserialize(
-                                getOneConfigFile(conn, TableKeys.HARDWARE_CONFIG), HardwareConfig.class);
+                                getOneConfigFile(conn, GlobalKeys.HARDWARE_CONFIG), HardwareConfig.class);
             } catch (IOException e) {
                 logger.error("Could not deserialize hardware config! Loading defaults");
                 hardwareConfig = new HardwareConfig();
@@ -219,7 +272,7 @@ public class SqlConfigProvider extends ConfigProvider {
             try {
                 hardwareSettings =
                         JacksonUtils.deserialize(
-                                getOneConfigFile(conn, TableKeys.HARDWARE_SETTINGS), HardwareSettings.class);
+                                getOneConfigFile(conn, GlobalKeys.HARDWARE_SETTINGS), HardwareSettings.class);
             } catch (IOException e) {
                 logger.error("Could not deserialize hardware settings! Loading defaults");
                 hardwareSettings = new HardwareSettings();
@@ -228,7 +281,7 @@ public class SqlConfigProvider extends ConfigProvider {
             try {
                 networkConfig =
                         JacksonUtils.deserialize(
-                                getOneConfigFile(conn, TableKeys.NETWORK_CONFIG), NetworkConfig.class);
+                                getOneConfigFile(conn, GlobalKeys.NETWORK_CONFIG), NetworkConfig.class);
             } catch (IOException e) {
                 logger.error("Could not deserialize network config! Loading defaults");
                 networkConfig = new NetworkConfig();
@@ -237,7 +290,7 @@ public class SqlConfigProvider extends ConfigProvider {
             try {
                 atfl =
                         JacksonUtils.deserialize(
-                                getOneConfigFile(conn, TableKeys.ATFL_CONFIG_FILE), AprilTagFieldLayout.class);
+                                getOneConfigFile(conn, GlobalKeys.ATFL_CONFIG_FILE), AprilTagFieldLayout.class);
             } catch (IOException e) {
                 logger.error("Could not deserialize apriltag layout! Loading defaults");
                 try {
@@ -271,12 +324,15 @@ public class SqlConfigProvider extends ConfigProvider {
         PreparedStatement query = null;
         try {
             query =
-                    conn.prepareStatement("SELECT contents FROM global where filename=\"" + filename + "\"");
+                    conn.prepareStatement(
+                            String.format(
+                                    "SELECT %s FROM %s WHERE %s = \"%s\"",
+                                    Columns.GLB_CONTENTS, Tables.GLOBAL, Columns.GLB_FILENAME, filename));
 
             var result = query.executeQuery();
 
             while (result.next()) {
-                return result.getString("contents");
+                return result.getString(Columns.GLB_CONTENTS);
             }
         } catch (SQLException e) {
             logger.error("SQL Err getting file " + filename, e);
@@ -295,8 +351,14 @@ public class SqlConfigProvider extends ConfigProvider {
         try {
             // Replace this camera's row with the new settings
             var sqlString =
-                    "REPLACE INTO cameras (unique_name, config_json, drivermode_json, pipeline_jsons) VALUES "
+                    String.format(
-                            + "(?,?,?,?);";
+                            "REPLACE INTO %s (%s, %s, %s, %s, %s) VALUES (?,?,?,?,?);",
+                            Tables.CAMERAS,
+                            Columns.CAM_UNIQUE_NAME,
+                            Columns.CAM_CONFIG_JSON,
+                            Columns.CAM_DRIVERMODE_JSON,
+                            Columns.CAM_OTHERPATHS_JSON,
+                            Columns.CAM_PIPELINE_JSONS);
 
             for (var c : config.getCameraConfigurations().entrySet()) {
                 PreparedStatement statement = conn.prepareStatement(sqlString);
@@ -305,6 +367,7 @@ public class SqlConfigProvider extends ConfigProvider {
                 statement.setString(1, c.getKey());
                 statement.setString(2, JacksonUtils.serializeToString(config));
                 statement.setString(3, JacksonUtils.serializeToString(config.driveModeSettings));
+                statement.setString(4, JacksonUtils.serializeToString(config.otherPaths));
 
                 // Serializing a list of abstract classes sucks. Instead, make it into an array
                 // of strings, which we can later unpack back into individual settings
@@ -321,7 +384,7 @@ public class SqlConfigProvider extends ConfigProvider {
                                         })
                                 .filter(Objects::nonNull)
                                 .collect(Collectors.toList());
-                statement.setString(4, JacksonUtils.serializeToString(settings));
+                statement.setString(5, JacksonUtils.serializeToString(settings));
 
                 statement.executeUpdate();
             }
@@ -340,36 +403,68 @@ public class SqlConfigProvider extends ConfigProvider {
         ps.setString(2, value);
     }
 
+    // NOTE to Future Developers:
+    // These booleans form a mechanism to prevent saveGlobal() and
+    // saveOneFile() from stepping on each other's toes. Both write
+    // to the database on disk, and both write to the same keys, but
+    // they use different sources. Generally, if the user has done something
+    // to trigger saveOneFile() to get called, it implies they want that
+    // configuration, and not whatever is in RAM right now (which is what
+    // saveGlobal() uses to write). Therefor, once saveOneFile() is invoked,
+    // we record which entry was overwritten in the database and prevent
+    // overwriting it when saveGlobal() is invoked (likely by the shutdown
+    // that should almost almost almost happen right after saveOneFile() is
+    // invoked).
+    //
+    // In the future, this may not be needed. A better architecture would involve
+    // manipulating the RAM representation of configuration when new .json files
+    // are uploaded in the UI, and eliminate all other usages of saveOneFile().
+    // But, seeing as it's Dec 28 and kickoff is nigh, we put this here and moved on.
+    // Thank you for coming to my TED talk.
+    private boolean skipSavingHWCfg = false;
+    private boolean skipSavingHWSet = false;
+    private boolean skipSavingNWCfg = false;
+    private boolean skipSavingAPRTG = false;
+
     private void saveGlobal(Connection conn) {
         PreparedStatement statement1 = null;
         PreparedStatement statement2 = null;
         PreparedStatement statement3 = null;
         try {
             // Replace this camera's row with the new settings
-            var sqlString = "REPLACE INTO global (filename, contents) VALUES " + "(?,?);";
+            var sqlString =
+                    String.format(
+                            "REPLACE INTO %s (%s, %s) VALUES (?,?);",
+                            Tables.GLOBAL, Columns.GLB_FILENAME, Columns.GLB_CONTENTS);
 
-            statement1 = conn.prepareStatement(sqlString);
+            if (!skipSavingHWSet) {
-            addFile(
+                statement1 = conn.prepareStatement(sqlString);
-                    statement1,
+                addFile(
-                    TableKeys.HARDWARE_SETTINGS,
+                        statement1,
-                    JacksonUtils.serializeToString(config.getHardwareSettings()));
+                        GlobalKeys.HARDWARE_SETTINGS,
-            statement1.executeUpdate();
+                        JacksonUtils.serializeToString(config.getHardwareSettings()));
+                statement1.executeUpdate();
+            }
 
-            statement2 = conn.prepareStatement(sqlString);
+            if (!skipSavingNWCfg) {
-            addFile(
+                statement2 = conn.prepareStatement(sqlString);
-                    statement2,
+                addFile(
-                    TableKeys.NETWORK_CONFIG,
+                        statement2,
-                    JacksonUtils.serializeToString(config.getNetworkConfig()));
+                        GlobalKeys.NETWORK_CONFIG,
-            statement2.executeUpdate();
+                        JacksonUtils.serializeToString(config.getNetworkConfig()));
-            statement2.close();
+                statement2.executeUpdate();
+                statement2.close();
+            }
 
-            statement3 = conn.prepareStatement(sqlString);
+            if (!skipSavingHWCfg) {
-            addFile(
+                statement3 = conn.prepareStatement(sqlString);
-                    statement3,
+                addFile(
-                    TableKeys.HARDWARE_CONFIG,
+                        statement3,
-                    JacksonUtils.serializeToString(config.getHardwareConfig()));
+                        GlobalKeys.HARDWARE_CONFIG,
-            statement3.executeUpdate();
+                        JacksonUtils.serializeToString(config.getHardwareConfig()));
-            statement3.close();
+                statement3.executeUpdate();
+                statement3.close();
+            }
 
         } catch (SQLException | IOException e) {
             logger.error("Err saving global", e);
@@ -400,7 +495,10 @@ public class SqlConfigProvider extends ConfigProvider {
             }
 
             // Replace this camera's row with the new settings
-            var sqlString = "REPLACE INTO global (filename, contents) VALUES " + "(?,?);";
+            var sqlString =
+                    String.format(
+                            "REPLACE INTO %s (%s, %s) VALUES (?,?);",
+                            Tables.GLOBAL, Columns.GLB_FILENAME, Columns.GLB_CONTENTS);
 
             statement1 = conn.prepareStatement(sqlString);
             addFile(statement1, fname, Files.readString(path));
@@ -428,22 +526,26 @@ public class SqlConfigProvider extends ConfigProvider {
 
     @Override
     public boolean saveUploadedHardwareConfig(Path uploadPath) {
-        return saveOneFile(TableKeys.HARDWARE_CONFIG, uploadPath);
+        skipSavingHWCfg = true;
+        return saveOneFile(GlobalKeys.HARDWARE_CONFIG, uploadPath);
     }
 
     @Override
     public boolean saveUploadedHardwareSettings(Path uploadPath) {
-        return saveOneFile(TableKeys.HARDWARE_SETTINGS, uploadPath);
+        skipSavingHWSet = true;
+        return saveOneFile(GlobalKeys.HARDWARE_SETTINGS, uploadPath);
     }
 
     @Override
     public boolean saveUploadedNetworkConfig(Path uploadPath) {
-        return saveOneFile(TableKeys.NETWORK_CONFIG, uploadPath);
+        skipSavingNWCfg = true;
+        return saveOneFile(GlobalKeys.NETWORK_CONFIG, uploadPath);
     }
 
     @Override
     public boolean saveUploadedAprilTagFieldLayout(Path uploadPath) {
-        return saveOneFile(TableKeys.ATFL_CONFIG_FILE, uploadPath);
+        skipSavingAPRTG = true;
+        return saveOneFile(GlobalKeys.ATFL_CONFIG_FILE, uploadPath);
     }
 
     private HashMap<String, CameraConfiguration> loadCameraConfigs(Connection conn) {
@@ -455,11 +557,13 @@ public class SqlConfigProvider extends ConfigProvider {
             query =
                     conn.prepareStatement(
                             String.format(
-                                    "SELECT %s, %s, %s, %s FROM cameras",
+                                    "SELECT %s, %s, %s, %s, %s FROM %s",
-                                    TableKeys.CAM_UNIQUE_NAME,
+                                    Columns.CAM_UNIQUE_NAME,
-                                    TableKeys.CONFIG_JSON,
+                                    Columns.CAM_CONFIG_JSON,
-                                    TableKeys.DRIVERMODE_JSON,
+                                    Columns.CAM_DRIVERMODE_JSON,
-                                    TableKeys.PIPELINE_JSONS));
+                                    Columns.CAM_OTHERPATHS_JSON,
+                                    Columns.CAM_PIPELINE_JSONS,
+                                    Tables.CAMERAS));
 
             var result = query.executeQuery();
 
@@ -467,16 +571,18 @@ public class SqlConfigProvider extends ConfigProvider {
             while (result.next()) {
                 List<String> dummyList = new ArrayList<>();
 
-                var uniqueName = result.getString(TableKeys.CAM_UNIQUE_NAME);
+                var uniqueName = result.getString(Columns.CAM_UNIQUE_NAME);
                 var config =
                         JacksonUtils.deserialize(
-                                result.getString(TableKeys.CONFIG_JSON), CameraConfiguration.class);
+                                result.getString(Columns.CAM_CONFIG_JSON), CameraConfiguration.class);
                 var driverMode =
                         JacksonUtils.deserialize(
-                                result.getString(TableKeys.DRIVERMODE_JSON), DriverModePipelineSettings.class);
+                                result.getString(Columns.CAM_DRIVERMODE_JSON), DriverModePipelineSettings.class);
+                var otherPaths =
+                        JacksonUtils.deserialize(result.getString(Columns.CAM_OTHERPATHS_JSON), String[].class);
                 List<?> pipelineSettings =
                         JacksonUtils.deserialize(
-                                result.getString(TableKeys.PIPELINE_JSONS), dummyList.getClass());
+                                result.getString(Columns.CAM_PIPELINE_JSONS), dummyList.getClass());
 
                 List<CVPipelineSettings> loadedSettings = new ArrayList<>();
                 for (var str : pipelineSettings) {
@@ -487,6 +593,7 @@ public class SqlConfigProvider extends ConfigProvider {
 
                 config.pipelineSettings = loadedSettings;
                 config.driveModeSettings = driverMode;
+                config.otherPaths = otherPaths;
                 loadedConfigurations.put(uniqueName, config);
             }
         } catch (SQLException | IOException e) {

photon-core/src/main/java/org/photonvision/common/dataflow/DataChangeDestination.java

@@ -26,6 +26,7 @@ public enum DataChangeDestination {
     DCD_ACTIVEPIPELINESETTINGS,
     DCD_GENSETTINGS,
     DCD_UI,
+    DCD_WEBSERVER,
     DCD_OTHER;
 
     public static final List<DataChangeDestination> AllDestinations =

photon-core/src/main/java/org/photonvision/common/dataflow/networktables/NTDataPublisher.java

@@ -22,8 +22,8 @@ import edu.wpi.first.networktables.NetworkTableEvent;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
+import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.dataflow.CVPipelineResultConsumer;
-import org.photonvision.common.dataflow.structures.Packet;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.networktables.NTTopicSet;
@@ -134,10 +134,11 @@ public class NTDataPublisher implements CVPipelineResultConsumer {
                         result.getLatencyMillis(),
                         TrackedTarget.simpleFromTrackedTargets(result.targets),
                         result.multiTagResult);
-        Packet packet = new Packet(simplified.getPacketSize());
-        simplified.populatePacket(packet);
 
-        ts.rawBytesEntry.set(packet.getData());
+        ts.resultPublisher.set(simplified, simplified.getPacketSize());
+        if (ConfigManager.getInstance().getConfig().getNetworkConfig().shouldPublishProto) {
+            ts.protoResultPublisher.set(simplified);
+        }
 
         ts.pipelineIndexPublisher.set(pipelineIndexSupplier.get());
         ts.driverModePublisher.set(driverModeSupplier.getAsBoolean());
@@ -183,7 +184,7 @@ public class NTDataPublisher implements CVPipelineResultConsumer {
                 && result.inputAndOutputFrame.frameStaticProperties.cameraCalibration != null) {
             var fsp = result.inputAndOutputFrame.frameStaticProperties;
             ts.cameraIntrinsicsPublisher.accept(fsp.cameraCalibration.getIntrinsicsArr());
-            ts.cameraDistortionPublisher.accept(fsp.cameraCalibration.getExtrinsicsArr());
+            ts.cameraDistortionPublisher.accept(fsp.cameraCalibration.getDistCoeffsArr());
         } else {
             ts.cameraIntrinsicsPublisher.accept(new double[] {});
             ts.cameraDistortionPublisher.accept(new double[] {});

photon-core/src/main/java/org/photonvision/common/dataflow/networktables/NetworkTablesManager.java

@@ -32,6 +32,7 @@ import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.configuration.NetworkConfig;
 import org.photonvision.common.dataflow.DataChangeService;
 import org.photonvision.common.dataflow.events.OutgoingUIEvent;
+import org.photonvision.common.hardware.HardwareManager;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.scripting.ScriptEventType;
@@ -91,6 +92,7 @@ public class NetworkTablesManager {
                                 event.connInfo.remote_port,
                                 event.connInfo.protocol_version);
                 logger.error(msg);
+                HardwareManager.getInstance().setNTConnected(false);
 
                 hasReportedConnectionFailure = true;
                 getInstance().broadcastConnectedStatus();
@@ -102,6 +104,7 @@ public class NetworkTablesManager {
                                 event.connInfo.remote_port,
                                 event.connInfo.protocol_version);
                 logger.info(msg);
+                HardwareManager.getInstance().setNTConnected(true);
 
                 hasReportedConnectionFailure = false;
                 ScriptManager.queueEvent(ScriptEventType.kNTConnected);

photon-core/src/main/java/org/photonvision/common/dataflow/statusLEDs/StatusLEDConsumer.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.dataflow.statusLEDs;
+
+import org.photonvision.common.dataflow.CVPipelineResultConsumer;
+import org.photonvision.common.hardware.HardwareManager;
+import org.photonvision.vision.pipeline.result.CVPipelineResult;
+
+public class StatusLEDConsumer implements CVPipelineResultConsumer {
+    private final int index;
+
+    public StatusLEDConsumer(int index) {
+        this.index = index;
+    }
+
+    @Override
+    public void accept(CVPipelineResult t) {
+        HardwareManager.getInstance().setTargetsVisibleStatus(this.index, t.hasTargets());
+    }
+}

photon-core/src/main/java/org/photonvision/common/dataflow/websocket/UIDataPublisher.java

@@ -52,6 +52,7 @@ public class UIDataPublisher implements CVPipelineResultConsumer {
             uiTargets.add(t.toHashMap());
         }
         dataMap.put("targets", uiTargets);
+        dataMap.put("classNames", result.objectDetectionClassNames);
 
         // Only send Multitag Results if they are present, similar to 3d pose
         if (result.multiTagResult.estimatedPose.isPresent) {

photon-core/src/main/java/org/photonvision/common/hardware/HardwareManager.java

@@ -20,7 +20,8 @@ package org.photonvision.common.hardware;
 import edu.wpi.first.networktables.IntegerPublisher;
 import edu.wpi.first.networktables.IntegerSubscriber;
 import java.io.IOException;
-import org.photonvision.common.ProgramStatus;
+import java.util.HashMap;
+import java.util.Map;
 import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.configuration.HardwareConfig;
 import org.photonvision.common.configuration.HardwareSettings;
@@ -32,6 +33,7 @@ import org.photonvision.common.hardware.metrics.MetricsManager;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.util.ShellExec;
+import org.photonvision.common.util.TimedTaskManager;
 
 public class HardwareManager {
     private static HardwareManager instance;
@@ -96,6 +98,10 @@ public class HardwareManager {
                         ? new StatusLED(hardwareConfig.statusRGBPins)
                         : null;
 
+        if (statusLED != null) {
+            TimedTaskManager.getInstance().addTask("StatusLEDUpdate", this::statusLEDUpdate, 150);
+        }
+
         var hasBrightnessRange = hardwareConfig.ledBrightnessRange.size() == 2;
         visionLED =
                 hardwareConfig.ledPins.isEmpty()
@@ -139,8 +145,7 @@ public class HardwareManager {
         logger.info("Shutting down LEDs...");
         if (visionLED != null) visionLED.setState(false);
 
-        logger.info("Force-flushing settings...");
+        ConfigManager.getInstance().onJvmExit();
-        ConfigManager.getInstance().saveToDisk();
     }
 
     public boolean restartDevice() {
@@ -160,21 +165,61 @@ public class HardwareManager {
         }
     }
 
-    public void setStatus(ProgramStatus status) {
+    // API's supporting status LEDs
-        switch (status) {
+
-            case UHOH:
+    private Map<Integer, Boolean> pipelineTargets = new HashMap<Integer, Boolean>();
-                // red flashing, green off
+    private boolean ntConnected = false;
-                break;
+    private boolean systemRunning = false;
-            case RUNNING:
+    private int blinkCounter = 0;
-                // red solid, green off
+
-                break;
+    public void setTargetsVisibleStatus(int pipelineIdx, boolean hasTargets) {
-            case RUNNING_NT:
+        pipelineTargets.put(pipelineIdx, hasTargets);
-                // red off, green solid
+    }
-                break;
+
-            case RUNNING_NT_TARGET:
+    public void setNTConnected(boolean isConnected) {
-                // red off, green flashing
+        this.ntConnected = isConnected;
-                break;
+    }
+
+    public void setRunning(boolean isRunning) {
+        this.systemRunning = isRunning;
+    }
+
+    private void statusLEDUpdate() {
+        // make blinky
+        boolean blinky = ((blinkCounter % 3) > 0);
+
+        // check if any pipeline has a visible target
+        boolean anyTarget = false;
+        for (var t : this.pipelineTargets.values()) {
+            if (t) {
+                anyTarget = true;
+            }
         }
+
+        if (this.systemRunning) {
+            if (!this.ntConnected) {
+                if (anyTarget) {
+                    // Blue Flashing
+                    statusLED.setRGB(false, false, blinky);
+                } else {
+                    // Yellow flashing
+                    statusLED.setRGB(blinky, blinky, false);
+                }
+            } else {
+                if (anyTarget) {
+                    // Blue
+                    statusLED.setRGB(false, false, blinky);
+                } else {
+                    // blinky green
+                    statusLED.setRGB(false, blinky, false);
+                }
+            }
+        } else {
+            // Faulted, not running... blinky red
+            statusLED.setRGB(blinky, false, false);
+        }
+
+        blinkCounter++;
     }
 
     public HardwareConfig getConfig() {

photon-core/src/main/java/org/photonvision/common/hardware/PiVersion.java

@@ -27,6 +27,7 @@ public enum PiVersion {
     ZERO_2_W("Raspberry Pi Zero 2"),
     PI_3("Pi 3"),
     PI_4("Pi 4"),
+    PI_5("Pi 5"),
     COMPUTE_MODULE_3("Compute Module 3"),
     UNKNOWN("UNKNOWN");
 

photon-core/src/main/java/org/photonvision/common/hardware/Platform.java

@@ -17,6 +17,7 @@
 
 package org.photonvision.common.hardware;
 
+import com.jogamp.common.os.Platform.OSType;
 import edu.wpi.first.util.RuntimeDetector;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -27,23 +28,33 @@ import org.photonvision.common.util.ShellExec;
 @SuppressWarnings("unused")
 public enum Platform {
     // WPILib Supported (JNI)
-    WINDOWS_64("Windows x64", false, OSType.WINDOWS, true),
+    WINDOWS_64("Windows x64", "winx64", false, OSType.WINDOWS, true),
-    LINUX_32("Linux x86", false, OSType.LINUX, true),
+    LINUX_32("Linux x86", "linuxx64", false, OSType.LINUX, true),
-    LINUX_64("Linux x64", false, OSType.LINUX, true),
+    LINUX_64("Linux x64", "linuxx64", false, OSType.LINUX, true),
     LINUX_RASPBIAN32(
-            "Linux Raspbian 32-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 32-bit image
+            "Linux Raspbian 32-bit",
+            "linuxarm32",
+            true,
+            OSType.LINUX,
+            true), // Raspberry Pi 3/4 with a 32-bit image
     LINUX_RASPBIAN64(
-            "Linux Raspbian 64-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 64-bit image
+            "Linux Raspbian 64-bit",
-    LINUX_AARCH64("Linux AARCH64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
+            "linuxarm64",
+            true,
+            OSType.LINUX,
+            true), // Raspberry Pi 3/4 with a 64-bit image
+    LINUX_RK3588_64("Linux AARCH 64-bit with RK3588", "linuxarm64", false, OSType.LINUX, true),
+    LINUX_AARCH64(
+            "Linux AARCH64", "linuxarm64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
 
     // PhotonVision Supported (Manual build/install)
-    LINUX_ARM32("Linux ARM32", false, OSType.LINUX, true), // ODROID XU4, C1+
+    LINUX_ARM32("Linux ARM32", "linuxarm32", false, OSType.LINUX, true), // ODROID XU4, C1+
-    LINUX_ARM64("Linux ARM64", false, OSType.LINUX, true), // ODROID C2, N2
+    LINUX_ARM64("Linux ARM64", "linuxarm64", false, OSType.LINUX, true), // ODROID C2, N2
 
     // Completely unsupported
-    WINDOWS_32("Windows x86", false, OSType.WINDOWS, false),
+    WINDOWS_32("Windows x86", "windowsx64", false, OSType.WINDOWS, false),
-    MACOS("Mac OS", false, OSType.MACOS, false),
+    MACOS("Mac OS", "osxuniversal", false, OSType.MACOS, false),
-    UNKNOWN("Unsupported Platform", false, OSType.UNKNOWN, false);
+    UNKNOWN("Unsupported Platform", "", false, OSType.UNKNOWN, false);
 
     private enum OSType {
         WINDOWS,
@@ -54,6 +65,7 @@ public enum Platform {
 
     private static final ShellExec shell = new ShellExec(true, false);
     public final String description;
+    public final String nativeLibraryFolderName;
     public final boolean isPi;
     public final OSType osType;
     public final boolean isSupported;
@@ -62,11 +74,17 @@ public enum Platform {
     private static final Platform currentPlatform = getCurrentPlatform();
     private static final boolean isRoot = checkForRoot();
 
-    Platform(String description, boolean isPi, OSType osType, boolean isSupported) {
+    Platform(
+            String description,
+            String nativeLibFolderName,
+            boolean isPi,
+            OSType osType,
+            boolean isSupported) {
         this.description = description;
         this.isPi = isPi;
         this.osType = osType;
         this.isSupported = isSupported;
+        this.nativeLibraryFolderName = nativeLibFolderName;
     }
 
     //////////////////////////////////////////////////////
@@ -77,6 +95,10 @@ public enum Platform {
         return currentPlatform.osType == OSType.LINUX;
     }
 
+    public static boolean isRK3588() {
+        return Platform.isOrangePi() || Platform.isCoolPi4b();
+    }
+
     public static boolean isRaspberryPi() {
         return currentPlatform.isPi;
     }
@@ -89,6 +111,10 @@ public enum Platform {
         }
     }
 
+    public static String getNativeLibraryFolderName() {
+        return currentPlatform.nativeLibraryFolderName;
+    }
+
     public static boolean isRoot() {
         return isRoot;
     }
@@ -165,7 +191,13 @@ public enum Platform {
                 return LINUX_32;
             } else if (RuntimeDetector.isArm64()) {
                 // TODO - os detection needed?
-                return LINUX_AARCH64;
+                if (isOrangePi()) {
+                    return LINUX_RK3588_64;
+                } else {
+                    return LINUX_AARCH64;
+                }
+            } else if (RuntimeDetector.isArm32()) {
+                return LINUX_ARM32;
             } else {
                 // Unknown or otherwise unsupported platform
                 return Platform.UNKNOWN;
@@ -181,6 +213,14 @@ public enum Platform {
         return fileHasText("/proc/cpuinfo", "Raspberry Pi");
     }
 
+    private static boolean isOrangePi() {
+        return fileHasText("/proc/device-tree/model", "Orange Pi 5");
+    }
+
+    private static boolean isCoolPi4b() {
+        return fileHasText("/proc/device-tree/model", "CoolPi 4B");
+    }
+
     private static boolean isJetsonSBC() {
         // https://forums.developer.nvidia.com/t/how-to-recognize-jetson-nano-device/146624
         return fileHasText("/proc/device-tree/model", "NVIDIA Jetson");
@@ -212,4 +252,9 @@ public enum Platform {
             return false;
         }
     }
+
+    public static boolean isWindows() {
+        var p = getCurrentPlatform();
+        return (p == WINDOWS_32 || p == WINDOWS_64);
+    }
 }

photon-core/src/main/java/org/photonvision/common/hardware/StatusLED.java

@@ -45,4 +45,14 @@ public class StatusLED {
             blueLED = new CustomGPIO(statusLedPins.get(2));
         }
     }
+
+    public void setRGB(boolean r, boolean g, boolean b) {
+        // Outputs are active-low, so invert the level applied
+        redLED.setState(!r);
+        redLED.setBrightness(r ? 0 : 100);
+        greenLED.setState(!g);
+        greenLED.setBrightness(g ? 0 : 100);
+        blueLED.setState(!b);
+        blueLED.setBrightness(b ? 0 : 100);
+    }
 }

photon-core/src/main/java/org/photonvision/common/hardware/metrics/MetricsManager.java

@@ -28,6 +28,7 @@ import org.photonvision.common.hardware.metrics.cmds.CmdBase;
 import org.photonvision.common.hardware.metrics.cmds.FileCmds;
 import org.photonvision.common.hardware.metrics.cmds.LinuxCmds;
 import org.photonvision.common.hardware.metrics.cmds.PiCmds;
+import org.photonvision.common.hardware.metrics.cmds.RK3588Cmds;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.util.ShellExec;
@@ -44,6 +45,8 @@ public class MetricsManager {
             cmds = new FileCmds();
         } else if (Platform.isRaspberryPi()) {
             cmds = new PiCmds(); // Pi's can use a hardcoded command set
+        } else if (Platform.isRK3588()) {
+            cmds = new RK3588Cmds(); // RK3588 chipset hardcoded command set
         } else if (Platform.isLinux()) {
             cmds = new LinuxCmds(); // Linux/Unix platforms assume a nominal command set
         } else {

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/LinuxCmds.java

@@ -22,7 +22,7 @@ import org.photonvision.common.configuration.HardwareConfig;
 public class LinuxCmds extends CmdBase {
     public void initCmds(HardwareConfig config) {
         // CPU
-        cpuMemoryCommand = "awk '/MemTotal:/ {print int($2 / 1000);}' /proc/meminfo";
+        cpuMemoryCommand = "free -m | awk 'FNR == 2 {print $2}'";
 
         // TODO: boards have lots of thermal devices. Hard to pick the CPU
 
@@ -32,7 +32,7 @@ public class LinuxCmds extends CmdBase {
         cpuUptimeCommand = "uptime -p | cut -c 4-";
 
         // RAM
-        ramUsageCommand = "awk '/MemAvailable:/ {print int($2 / 1000);}' /proc/meminfo";
+        ramUsageCommand = "free -m | awk 'FNR == 2 {print $3}'";
 
         // Disk
         diskUsageCommand = "df ./ --output=pcent | tail -n +2";

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/PiCmds.java

@@ -25,7 +25,6 @@ public class PiCmds extends LinuxCmds {
         super.initCmds(config);
 
         // CPU
-        cpuMemoryCommand = "vcgencmd get_mem arm | grep -Eo '[0-9]+'";
         cpuTemperatureCommand = "sed 's/.\\{3\\}$/.&/' /sys/class/thermal/thermal_zone0/temp";
         cpuThrottleReasonCmd =
                 "if   ((  $(( $(vcgencmd get_throttled | grep -Eo 0x[0-9a-fA-F]*) & 0x01 )) != 0x00 )); then echo \"LOW VOLTAGE\"; "

photon-core/src/main/java/org/photonvision/common/hardware/metrics/cmds/RK3588Cmds.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.hardware.metrics.cmds;
+
+import org.photonvision.common.configuration.HardwareConfig;
+
+public class RK3588Cmds extends LinuxCmds {
+    /** Applies pi-specific commands, ignoring any input configuration */
+    public void initCmds(HardwareConfig config) {
+        super.initCmds(config);
+
+        // CPU Temperature
+        /* The RK3588 chip has 7 thermal zones that can be accessed via:
+         *       /sys/class/thermal/thermal_zoneX/temp
+         * where X is an interger from 0 to 6.
+         *
+         * || Zone || Location    || Comments                                                ||
+         * |    0  |  soc         |  soc thermal (near the center of the chip)                |
+         * |    1  |  bigcore0    |  CPU Big Core A76_0/1 (CPU4 and CPU5)                     |
+         * |    2  |  bigcore1    |  CPU Big Core A76_2/3 (CPU6 and CPU7)                     |
+         * |    3  |  littlecore  |  CPU Small Core A55_0/1/2/3 (CPU0, CPU1, CPU2, and CPU3)  |
+         * |    4  |  center      |  also called PD_CENTER                                    |
+         * |    5  |  gpu         |  GPU                                                      |
+         * |    6  |  npu         |  NPU                                                      |
+         *
+         * Sources:
+         *   - http://forum.armsom.org/t/topic/51/3
+         *   - https://lore.kernel.org/lkml/7276280.TLKafQO6qx@archbook/
+         */
+        cpuTemperatureCommand =
+                "cat /sys/class/thermal/thermal_zone1/temp | awk '{printf \"%.1f\", $1/1000}'";
+    }
+}

photon-core/src/main/java/org/photonvision/common/logging/Logger.java

@@ -24,7 +24,8 @@ import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.function.Supplier;
 import org.apache.commons.lang3.tuple.Pair;
-import org.photonvision.common.configuration.ConfigManager;
+// import org.photonvision.common.configuration.ConfigManager;
+import org.photonvision.common.configuration.PathManager;
 import org.photonvision.common.dataflow.DataChangeService;
 import org.photonvision.common.dataflow.events.OutgoingUIEvent;
 import org.photonvision.common.util.TimedTaskManager;
@@ -103,8 +104,8 @@ public class Logger {
     static {
         currentAppenders.add(new ConsoleLogAppender());
         currentAppenders.add(uiLogAppender);
-        addFileAppender(ConfigManager.getInstance().getLogPath());
+        addFileAppender(PathManager.getInstance().getLogPath());
-        cleanLogs(ConfigManager.getInstance().getLogsDir());
+        cleanLogs(PathManager.getInstance().getLogsDir());
     }
 
     @SuppressWarnings("ResultOfMethodCallIgnored")
@@ -133,8 +134,7 @@ public class Logger {
         logFileList.removeIf(
                 (File arg0) -> {
                     try {
-                        logFileStartDateMap.put(
+                        logFileStartDateMap.put(arg0, PathManager.getInstance().logFnameToDate(arg0.getName()));
-                                arg0, ConfigManager.getInstance().logFnameToDate(arg0.getName()));
                         return false;
                     } catch (ParseException e) {
                         return true;

photon-core/src/main/java/org/photonvision/common/networking/NetworkManager.java

@@ -19,6 +19,10 @@ package org.photonvision.common.networking;
 
 import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.configuration.NetworkConfig;
+import org.photonvision.common.dataflow.DataChangeDestination;
+import org.photonvision.common.dataflow.DataChangeService;
+import org.photonvision.common.dataflow.DataChangeSource;
+import org.photonvision.common.dataflow.events.DataChangeEvent;
 import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
@@ -43,6 +47,7 @@ public class NetworkManager {
     public void initialize(boolean shouldManage) {
         isManaged = shouldManage && !networkingIsDisabled;
         if (!isManaged) {
+            logger.info("Network management is disabled.");
             return;
         }
 
@@ -147,5 +152,13 @@ public class NetworkManager {
 
     public void reinitialize() {
         initialize(ConfigManager.getInstance().getConfig().getNetworkConfig().shouldManage());
+
+        DataChangeService.getInstance()
+                .publishEvent(
+                        new DataChangeEvent<Boolean>(
+                                DataChangeSource.DCS_OTHER,
+                                DataChangeDestination.DCD_WEBSERVER,
+                                "restartServer",
+                                true));
     }
 }

photon-core/src/main/java/org/photonvision/common/util/ColorHelper.java

@@ -24,4 +24,8 @@ public class ColorHelper {
     public static Scalar colorToScalar(Color color) {
         return new Scalar(color.getBlue(), color.getGreen(), color.getRed());
     }
+
+    public static Scalar colorToScalar(Color color, double alpha) {
+        return new Scalar(color.getBlue(), color.getGreen(), color.getRed(), alpha);
+    }
 }

photon-core/src/main/java/org/photonvision/common/util/SerializationUtils.java

@@ -55,6 +55,8 @@ public final class SerializationUtils {
         ret.put("qy", transform.getRotation().getQuaternion().getY());
         ret.put("qz", transform.getRotation().getQuaternion().getZ());
 
+        ret.put("angle_x", transform.getRotation().getX());
+        ret.put("angle_y", transform.getRotation().getY());
         ret.put("angle_z", transform.getRotation().getZ());
         return ret;
     }

photon-core/src/main/java/org/photonvision/common/util/TestUtils.java

@@ -39,7 +39,11 @@ import org.opencv.highgui.HighGui;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
 
 public class TestUtils {
+    private static boolean has_loaded = false;
+
     public static boolean loadLibraries() {
+        if (has_loaded) return true;
+
         NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
         WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
         WPIMathJNI.Helper.setExtractOnStaticLoad(false);
@@ -61,11 +65,13 @@ public class TestUtils {
                     "cscorejni",
                     "apriltagjni");
 
-            return true;
+            has_loaded = true;
         } catch (IOException e) {
             e.printStackTrace();
-            return false;
+            has_loaded = false;
         }
+
+        return has_loaded;
     }
 
     @SuppressWarnings("unused")
@@ -138,6 +144,24 @@ public class TestUtils {
         }
     }
 
+    public enum WPI2024Images {
+        kBackAmpZone_117in,
+        kSpeakerCenter_143in;
+
+        public static double FOV = 68.5;
+
+        public final Path path;
+
+        Path getPath() {
+            var filename = this.toString().substring(1);
+            return Path.of("2024", filename + ".jpg");
+        }
+
+        WPI2024Images() {
+            this.path = getPath();
+        }
+    }
+
     public enum WPI2023Apriltags {
         k162_36_Angle,
         k162_36_Straight,
@@ -356,6 +380,10 @@ public class TestUtils {
         return getCoeffs(LIFECAM_480P_CAL_FILE, testMode);
     }
 
+    public static CameraCalibrationCoefficients get2023LifeCamCoeffs(boolean testMode) {
+        return getCoeffs(LIFECAM_1280P_CAL_FILE, testMode);
+    }
+
     public static CameraCalibrationCoefficients getLaptop() {
         return getCoeffs("laptop.json", true);
     }
@@ -389,8 +417,4 @@ public class TestUtils {
                 .resolve("testimages")
                 .resolve(WPI2022Image.kTerminal22ft6in.path);
     }
-
-    public static CameraCalibrationCoefficients get2023LifeCamCoeffs(boolean testMode) {
-        return getCoeffs(LIFECAM_1280P_CAL_FILE, testMode);
-    }
 }

photon-core/src/main/java/org/photonvision/common/util/file/JacksonUtils.java

@@ -32,6 +32,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.HashMap;
+import java.util.Map;
 
 public class JacksonUtils {
     public static class UIMap extends HashMap<String, Object> {}
@@ -61,6 +62,19 @@ public class JacksonUtils {
         saveJsonString(json, path, forceSync);
     }
 
+    public static <T> T deserialize(Map<?, ?> s, Class<T> ref) throws IOException {
+        PolymorphicTypeValidator ptv =
+                BasicPolymorphicTypeValidator.builder().allowIfBaseType(ref).build();
+        ObjectMapper objectMapper =
+                JsonMapper.builder()
+                        .configure(JsonReadFeature.ALLOW_JAVA_COMMENTS, true)
+                        .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                        .activateDefaultTyping(ptv, ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT)
+                        .build();
+
+        return objectMapper.convertValue(s, ref);
+    }
+
     public static <T> T deserialize(String s, Class<T> ref) throws IOException {
         PolymorphicTypeValidator ptv =
                 BasicPolymorphicTypeValidator.builder().allowIfBaseType(ref).build();

photon-core/src/main/java/org/photonvision/common/util/math/MathUtils.java

@@ -23,10 +23,12 @@ import edu.wpi.first.math.geometry.CoordinateSystem;
 import edu.wpi.first.math.geometry.Pose3d;
 import edu.wpi.first.math.geometry.Rotation3d;
 import edu.wpi.first.math.geometry.Transform3d;
+import edu.wpi.first.math.geometry.Translation3d;
 import edu.wpi.first.math.util.Units;
 import edu.wpi.first.util.WPIUtilJNI;
 import java.util.Arrays;
 import java.util.List;
+import org.opencv.core.Core;
 import org.opencv.core.Mat;
 
 public class MathUtils {
@@ -198,4 +200,23 @@ public class MathUtils {
         var axis = rotation.getAxis().times(angle);
         rvecOutput.put(0, 0, axis.getData());
     }
+
+    /**
+     * Convert an Opencv rvec+tvec pair to a Pose3d.
+     *
+     * @param rVec Axis-angle rotation vector, where norm(rVec) is the angle about a unit vector in
+     *     the direction of rVec
+     * @param tVec 3D translation
+     * @return Pose3d representing the same rigid transform
+     */
+    public static Pose3d opencvRTtoPose3d(Mat rVec, Mat tVec) {
+        Translation3d translation =
+                new Translation3d(tVec.get(0, 0)[0], tVec.get(1, 0)[0], tVec.get(2, 0)[0]);
+        Rotation3d rotation =
+                new Rotation3d(
+                        VecBuilder.fill(rVec.get(0, 0)[0], rVec.get(1, 0)[0], rVec.get(2, 0)[0]),
+                        Core.norm(rVec));
+
+        return new Pose3d(translation, rotation);
+    }
 }

photon-core/src/main/java/org/photonvision/jni/PhotonJNICommon.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.jni;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+import org.photonvision.common.hardware.Platform;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+public abstract class PhotonJNICommon {
+    public abstract boolean isLoaded();
+
+    public abstract void setLoaded(boolean state);
+
+    protected static Logger logger = null;
+
+    protected static synchronized void forceLoad(
+            PhotonJNICommon instance, Class<?> clazz, List<String> libraries) throws IOException {
+        if (instance.isLoaded()) return;
+        if (logger == null) logger = new Logger(clazz, LogGroup.Camera);
+
+        for (var libraryName : libraries) {
+            try {
+                // We always extract the shared object (we could hash each so, but that's a lot of work)
+                var arch_name = Platform.getNativeLibraryFolderName();
+                var nativeLibName = System.mapLibraryName(libraryName);
+                var in = clazz.getResourceAsStream("/nativelibraries/" + arch_name + "/" + nativeLibName);
+
+                if (in == null) {
+                    instance.setLoaded(false);
+                    return;
+                }
+
+                // It's important that we don't mangle the names of these files on Windows at least
+                File temp = new File(System.getProperty("java.io.tmpdir"), nativeLibName);
+                FileOutputStream fos = new FileOutputStream(temp);
+
+                int read = -1;
+                byte[] buffer = new byte[1024];
+                while ((read = in.read(buffer)) != -1) {
+                    fos.write(buffer, 0, read);
+                }
+                fos.close();
+                in.close();
+
+                System.load(temp.getAbsolutePath());
+
+                logger.info("Successfully loaded shared object " + temp.getName());
+
+            } catch (UnsatisfiedLinkError e) {
+                logger.error("Couldn't load shared object " + libraryName, e);
+                e.printStackTrace();
+                // logger.error(System.getProperty("java.library.path"));
+                break;
+            }
+        }
+        instance.setLoaded(true);
+    }
+
+    protected static synchronized void forceLoad(
+            PhotonJNICommon instance, Class<?> clazz, String libraryName) throws IOException {
+        forceLoad(instance, clazz, List.of(libraryName));
+    }
+}

photon-core/src/main/java/org/photonvision/jni/RknnDetectorJNI.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.jni;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+import org.photonvision.common.util.TestUtils;
+import org.photonvision.rknn.RknnJNI;
+import org.photonvision.rknn.RknnJNI.RknnResult;
+import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult;
+
+public class RknnDetectorJNI extends PhotonJNICommon {
+    private static final Logger logger = new Logger(RknnDetectorJNI.class, LogGroup.General);
+    private boolean isLoaded;
+    private static RknnDetectorJNI instance = null;
+
+    private RknnDetectorJNI() {
+        isLoaded = false;
+    }
+
+    public static RknnDetectorJNI getInstance() {
+        if (instance == null) instance = new RknnDetectorJNI();
+
+        return instance;
+    }
+
+    public static synchronized void forceLoad() throws IOException {
+        TestUtils.loadLibraries();
+
+        forceLoad(getInstance(), RknnDetectorJNI.class, List.of("rga", "rknnrt", "rknn_jni"));
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return isLoaded;
+    }
+
+    @Override
+    public void setLoaded(boolean state) {
+        isLoaded = state;
+    }
+
+    public static class RknnObjectDetector {
+        long objPointer = -1;
+        private List<String> labels;
+        private final Object lock = new Object();
+
+        private static final CopyOnWriteArrayList<Long> detectors = new CopyOnWriteArrayList<>();
+
+        public RknnObjectDetector(String modelPath, List<String> labels) {
+            synchronized (lock) {
+                objPointer = RknnJNI.create(modelPath, labels.size());
+                detectors.add(objPointer);
+                System.out.println(
+                        "Created " + objPointer + "! Detectors: " + Arrays.toString(detectors.toArray()));
+            }
+            this.labels = labels;
+        }
+
+        public List<String> getClasses() {
+            return labels;
+        }
+
+        /**
+         * Detect forwards using this model
+         *
+         * @param in The image to process
+         * @param nmsThresh Non-maximum supression threshold. Probably should not change
+         * @param boxThresh Minimum confidence for a box to be added. Basically just confidence
+         *     threshold
+         */
+        public List<NeuralNetworkPipeResult> detect(CVMat in, double nmsThresh, double boxThresh) {
+            RknnResult[] ret;
+            synchronized (lock) {
+                // We can technically be asked to detect and the lock might be acquired _after_ release has
+                // been called. This would mean objPointer would be invalid which would call everything to
+                // explode.
+                if (objPointer > 0) {
+                    ret = RknnJNI.detect(objPointer, in.getMat().getNativeObjAddr(), nmsThresh, boxThresh);
+                } else {
+                    logger.warn("Detect called after destroy -- giving up");
+                    return List.of();
+                }
+            }
+            if (ret == null) {
+                return List.of();
+            }
+            return List.of(ret).stream()
+                    .map(it -> new NeuralNetworkPipeResult(it.rect, it.class_id, it.conf))
+                    .collect(Collectors.toList());
+        }
+
+        public void release() {
+            synchronized (lock) {
+                if (objPointer > 0) {
+                    RknnJNI.destroy(objPointer);
+                    detectors.remove(objPointer);
+                    System.out.println(
+                            "Killed " + objPointer + "! Detectors: " + Arrays.toString(detectors.toArray()));
+                    objPointer = -1;
+                } else {
+                    logger.error("RKNN Detector has already been destroyed!");
+                }
+            }
+        }
+    }
+
+    // public static void createRknnDetector() {
+    //     objPointer =
+    //             RknnJNI.create(
+    //                     NeuralNetworkModelManager.getInstance()
+    //                             .getDefaultRknnModel()
+    //                             .getAbsolutePath()
+    //                             .toString(),
+    //                     NeuralNetworkModelManager.getInstance().getLabels().size());
+    // }
+}

photon-core/src/main/java/org/photonvision/mrcal/MrCalJNILoader.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.mrcal;
+
+import java.io.IOException;
+import java.util.List;
+import org.photonvision.common.hardware.Platform;
+import org.photonvision.common.util.TestUtils;
+import org.photonvision.jni.PhotonJNICommon;
+
+public class MrCalJNILoader extends PhotonJNICommon {
+    private boolean isLoaded;
+    private static MrCalJNILoader instance = null;
+
+    private MrCalJNILoader() {
+        isLoaded = false;
+    }
+
+    public static synchronized MrCalJNILoader getInstance() {
+        if (instance == null) instance = new MrCalJNILoader();
+
+        return instance;
+    }
+
+    public static synchronized void forceLoad() throws IOException {
+        // Force load opencv
+        TestUtils.loadLibraries();
+
+        // Library naming is dumb and has "lib" appended for Windows when it ought not to
+        if (Platform.isWindows()) {
+            // Order is correct to match dependencies of libraries
+            forceLoad(
+                    MrCalJNILoader.getInstance(),
+                    MrCalJNILoader.class,
+                    List.of(
+                            "libamd",
+                            "libcamd",
+                            "libcolamd",
+                            "libccolamd",
+                            "openblas",
+                            "libgcc_s_seh-1",
+                            "libquadmath-0",
+                            "libgfortran-5",
+                            "liblapack",
+                            "libcholmod",
+                            "mrcal_jni"));
+        } else {
+            // Nothing else to do on linux
+            forceLoad(MrCalJNILoader.getInstance(), MrCalJNILoader.class, List.of("mrcal_jni"));
+        }
+
+        if (!MrCalJNILoader.getInstance().isLoaded()) {
+            throw new IOException("Unable to load mrcal JNI!");
+        }
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return isLoaded;
+    }
+
+    @Override
+    public void setLoaded(boolean state) {
+        isLoaded = state;
+    }
+}

photon-core/src/main/java/org/photonvision/raspi/LibCameraJNI.java

@@ -1,191 +0,0 @@
-/*
- * Copyright (C) Photon Vision.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.photonvision.raspi;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.photonvision.common.logging.LogGroup;
-import org.photonvision.common.logging.Logger;
-
-public class LibCameraJNI {
-    private static boolean libraryLoaded = false;
-    private static final Logger logger = new Logger(LibCameraJNI.class, LogGroup.Camera);
-
-    public static final Object CAMERA_LOCK = new Object();
-
-    public static synchronized void forceLoad() throws IOException {
-        if (libraryLoaded) return;
-
-        try {
-            File libDirectory = Path.of("lib/").toFile();
-            if (!libDirectory.exists()) {
-                Files.createDirectory(libDirectory.toPath()).toFile();
-            }
-
-            // We always extract the shared object (we could hash each so, but that's a lot of work)
-            URL resourceURL = LibCameraJNI.class.getResource("/nativelibraries/libphotonlibcamera.so");
-            File libFile = Path.of("lib/libphotonlibcamera.so").toFile();
-            try (InputStream in = resourceURL.openStream()) {
-                if (libFile.exists()) Files.delete(libFile.toPath());
-                Files.copy(in, libFile.toPath());
-            } catch (Exception e) {
-                logger.error("Could not extract the native library!");
-            }
-            System.load(libFile.getAbsolutePath());
-
-            libraryLoaded = true;
-            logger.info("Successfully loaded libpicam shared object");
-        } catch (UnsatisfiedLinkError e) {
-            logger.error("Couldn't load libpicam shared object");
-            e.printStackTrace();
-        }
-    }
-
-    public enum SensorModel {
-        Disconnected,
-        OV5647, // Picam v1
-        IMX219, // Picam v2
-        IMX477, // Picam HQ
-        OV9281,
-        OV7251,
-        Unknown;
-
-        public String getFriendlyName() {
-            switch (this) {
-                case Disconnected:
-                    return "Disconnected Camera";
-                case OV5647:
-                    return "Camera Module v1";
-                case IMX219:
-                    return "Camera Module v2";
-                case IMX477:
-                    return "HQ Camera";
-                case OV9281:
-                    return "OV9281";
-                case OV7251:
-                    return "OV7251";
-                case Unknown:
-                default:
-                    return "Unknown Camera";
-            }
-        }
-    }
-
-    public static SensorModel getSensorModel() {
-        int model = getSensorModelRaw();
-        return SensorModel.values()[model];
-    }
-
-    public static boolean isSupported() {
-        return libraryLoaded
-                // && getSensorModel() != PicamJNI.SensorModel.Disconnected
-                // && Platform.isRaspberryPi()
-                && isLibraryWorking();
-    }
-
-    private static native boolean isLibraryWorking();
-
-    public static native int getSensorModelRaw();
-
-    // ======================================================== //
-
-    /**
-     * Creates a new runner with a given width/height/fps
-     *
-     * @param width Camera video mode width in pixels
-     * @param height Camera video mode height in pixels
-     * @param fps Camera video mode FPS
-     * @return success of creating a camera object
-     */
-    public static native boolean createCamera(int width, int height, int rotation);
-
-    /**
-     * Starts the camera thresholder and display threads running. Make sure that this function is
-     * called synchronously with stopCamera and returnFrame!
-     */
-    public static native boolean startCamera();
-
-    /** Stops the camera runner. Make sure to call prior to destroying the camera! */
-    public static native boolean stopCamera();
-
-    // Destroy all native resources associated with a camera. Ensure stop is called prior!
-    public static native boolean destroyCamera();
-
-    // ======================================================== //
-
-    // Set thresholds on [0..1]
-    public static native boolean setThresholds(
-            double hl, double sl, double vl, double hu, double su, double vu, boolean hueInverted);
-
-    public static native boolean setAutoExposure(boolean doAutoExposure);
-
-    // Exposure time, in microseconds
-    public static native boolean setExposure(int exposureUs);
-
-    // Set brightness on [-1, 1]
-    public static native boolean setBrightness(double brightness);
-
-    // Unknown ranges for red and blue AWB gain
-    public static native boolean setAwbGain(double red, double blue);
-
-    /**
-     * Get the time when the first pixel exposure was started, in the same timebase as libcamera gives
-     * the frame capture time. Units are nanoseconds.
-     */
-    public static native long getFrameCaptureTime();
-
-    /**
-     * Get the current time, in the same timebase as libcamera gives the frame capture time. Units are
-     * nanoseconds.
-     */
-    public static native long getLibcameraTimestamp();
-
-    public static native long setFramesToCopy(boolean copyIn, boolean copyOut);
-
-    // Analog gain multiplier to apply to all color channels, on [1, Big Number]
-    public static native boolean setAnalogGain(double analog);
-
-    /** Block until a new frame is available from native code. */
-    public static native boolean awaitNewFrame();
-
-    /**
-     * Get a pointer to the most recent color mat generated. Call this immediately after
-     * awaitNewFrame, and call only once per new frame!
-     */
-    public static native long takeColorFrame();
-
-    /**
-     * Get a pointer to the most recent processed mat generated. Call this immediately after
-     * awaitNewFrame, and call only once per new frame!
-     */
-    public static native long takeProcessedFrame();
-
-    /**
-     * Set the GPU processing type we should do. Enum of [none, HSV, greyscale, adaptive threshold].
-     */
-    public static native boolean setGpuProcessType(int type);
-
-    public static native int getGpuProcessType();
-
-    // Release a frame pointer back to the libcamera driver code to be filled again */
-    // public static native long returnFrame(long frame);
-}

photon-core/src/main/java/org/photonvision/raspi/LibCameraJNILoader.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.raspi;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+/**
+ * Helper for extracting photon-libcamera-gl-driver shared library files. TODO: Refactor to use
+ * PhotonJNICommon
+ */
+public class LibCameraJNILoader {
+    private static boolean libraryLoaded = false;
+    private static final Logger logger = new Logger(LibCameraJNILoader.class, LogGroup.Camera);
+
+    public static synchronized void forceLoad() throws IOException {
+        if (libraryLoaded) return;
+
+        var libraryName = "photonlibcamera";
+
+        try {
+            // We always extract the shared object (we could hash each so, but that's a lot of work)
+            var arch_name = "linuxarm64";
+            var nativeLibName = System.mapLibraryName(libraryName);
+            var resourcePath = "/nativelibraries/" + arch_name + "/" + nativeLibName;
+            var in = LibCameraJNILoader.class.getResourceAsStream(resourcePath);
+
+            if (in == null) {
+                logger.error("Failed to find internal native library at path " + resourcePath);
+                libraryLoaded = false;
+                return;
+            }
+
+            // It's important that we don't mangle the names of these files on Windows at least
+            File temp = new File(System.getProperty("java.io.tmpdir"), nativeLibName);
+            FileOutputStream fos = new FileOutputStream(temp);
+
+            int read = -1;
+            byte[] buffer = new byte[1024];
+            while ((read = in.read(buffer)) != -1) {
+                fos.write(buffer, 0, read);
+            }
+            fos.close();
+            in.close();
+
+            System.load(temp.getAbsolutePath());
+
+            logger.info("Successfully loaded shared object " + temp.getName());
+
+        } catch (UnsatisfiedLinkError e) {
+            logger.error("Couldn't load shared object " + libraryName, e);
+            e.printStackTrace();
+            // logger.error(System.getProperty("java.library.path"));
+        }
+        libraryLoaded = true;
+    }
+
+    public static boolean isSupported() {
+        return libraryLoaded && LibCameraJNI.isSupported();
+    }
+}

photon-core/src/main/java/org/photonvision/vision/calibration/BoardObservation.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.calibration;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import edu.wpi.first.math.geometry.Pose3d;
+import java.util.List;
+import org.opencv.core.Point;
+import org.opencv.core.Point3;
+
+public final class BoardObservation {
+    // Expected feature 3d location in the camera frame
+    @JsonProperty("locationInObjectSpace")
+    public List<Point3> locationInObjectSpace;
+
+    // Observed location in pixel space
+    @JsonProperty("locationInImageSpace")
+    public List<Point> locationInImageSpace;
+
+    // (measured location in pixels) - (expected from FK)
+    @JsonProperty("reprojectionErrors")
+    public List<Point> reprojectionErrors;
+
+    // Solver optimized board poses
+    @JsonProperty("optimisedCameraToObject")
+    public Pose3d optimisedCameraToObject;
+
+    // If we should use this observation when re-calculating camera calibration
+    @JsonProperty("includeObservationInCalibration")
+    public boolean includeObservationInCalibration;
+
+    @JsonProperty("snapshotName")
+    public String snapshotName;
+
+    @JsonProperty("snapshotData")
+    public JsonImageMat snapshotData;
+
+    @JsonCreator
+    public BoardObservation(
+            @JsonProperty("locationInObjectSpace") List<Point3> locationInObjectSpace,
+            @JsonProperty("locationInImageSpace") List<Point> locationInImageSpace,
+            @JsonProperty("reprojectionErrors") List<Point> reprojectionErrors,
+            @JsonProperty("optimisedCameraToObject") Pose3d optimisedCameraToObject,
+            @JsonProperty("includeObservationInCalibration") boolean includeObservationInCalibration,
+            @JsonProperty("snapshotName") String snapshotName,
+            @JsonProperty("snapshotData") JsonImageMat snapshotData) {
+        this.locationInObjectSpace = locationInObjectSpace;
+        this.locationInImageSpace = locationInImageSpace;
+        this.reprojectionErrors = reprojectionErrors;
+        this.optimisedCameraToObject = optimisedCameraToObject;
+        this.includeObservationInCalibration = includeObservationInCalibration;
+        this.snapshotName = snapshotName;
+        this.snapshotData = snapshotData;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/calibration/CameraCalibrationCoefficients.java

@@ -20,50 +20,91 @@ package org.photonvision.vision.calibration;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.JsonNode;
+import java.util.Arrays;
+import java.util.List;
 import org.opencv.core.Mat;
 import org.opencv.core.MatOfDouble;
 import org.opencv.core.Size;
 import org.photonvision.vision.opencv.Releasable;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class CameraCalibrationCoefficients implements Releasable {
     @JsonProperty("resolution")
     public final Size resolution;
 
     @JsonProperty("cameraIntrinsics")
-    public final JsonMat cameraIntrinsics;
+    public final JsonMatOfDouble cameraIntrinsics;
 
-    @JsonProperty("cameraExtrinsics")
+    @JsonProperty("distCoeffs")
-    @JsonAlias({"cameraExtrinsics", "distCoeffs"})
+    @JsonAlias({"distCoeffs", "distCoeffs"})
-    public final JsonMat distCoeffs;
+    public final JsonMatOfDouble distCoeffs;
 
-    @JsonProperty("perViewErrors")
+    @JsonProperty("observations")
-    public final double[] perViewErrors;
+    public final List<BoardObservation> observations;
 
-    @JsonProperty("standardDeviation")
+    @JsonProperty("calobjectWarp")
-    public final double standardDeviation;
+    public final double[] calobjectWarp;
+
+    @JsonProperty("calobjectSize")
+    public final Size calobjectSize;
+
+    @JsonProperty("calobjectSpacing")
+    public final double calobjectSpacing;
+
+    @JsonProperty("lensmodel")
+    public final CameraLensModel lensmodel;
 
     @JsonIgnore private final double[] intrinsicsArr = new double[9];
+    @JsonIgnore private final double[] distCoeffsArr = new double[5];
 
-    @JsonIgnore private final double[] extrinsicsArr = new double[5];
+    /**
-
+     * Contains all camera calibration data for a particular resolution of a camera. Designed for use
+     * with standard opencv camera calibration matrices. For details on the layout of camera
+     * intrinsics/distortion matrices, see:
+     * https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d
+     *
+     * @param resolution The resolution this applies to. We don't assume camera binning or try
+     *     rescaling calibration
+     * @param cameraIntrinsics Camera intrinsics parameters matrix, in the standard opencv form.
+     * @param distCoeffs Camera distortion coefficients array. Variable length depending on order of
+     *     distortion model
+     * @param calobjectWarp Board deformation parameters, for calibrators that can estimate that. See:
+     *     https://mrcal.secretsauce.net/formulation.html#board-deformation
+     * @param observations List of snapshots used to construct this calibration
+     * @param calobjectSize Dimensions of the object used to calibrate, in # of squares in
+     *     width/height
+     * @param calobjectSpacing Spacing between adjacent squares, in meters
+     */
     @JsonCreator
     public CameraCalibrationCoefficients(
             @JsonProperty("resolution") Size resolution,
-            @JsonProperty("cameraIntrinsics") JsonMat cameraIntrinsics,
+            @JsonProperty("cameraIntrinsics") JsonMatOfDouble cameraIntrinsics,
-            @JsonProperty("cameraExtrinsics") JsonMat distCoeffs,
+            @JsonProperty("distCoeffs") JsonMatOfDouble distCoeffs,
-            @JsonProperty("perViewErrors") double[] perViewErrors,
+            @JsonProperty("calobjectWarp") double[] calobjectWarp,
-            @JsonProperty("standardDeviation") double standardDeviation) {
+            @JsonProperty("observations") List<BoardObservation> observations,
+            @JsonProperty("calobjectSize") Size calobjectSize,
+            @JsonProperty("calobjectSpacing") double calobjectSpacing,
+            @JsonProperty("lensmodel") CameraLensModel lensmodel) {
         this.resolution = resolution;
         this.cameraIntrinsics = cameraIntrinsics;
         this.distCoeffs = distCoeffs;
-        this.perViewErrors = perViewErrors;
+        this.calobjectWarp = calobjectWarp;
-        this.standardDeviation = standardDeviation;
+        this.calobjectSize = calobjectSize;
+        this.calobjectSpacing = calobjectSpacing;
+        this.lensmodel = lensmodel;
+
+        // Legacy migration just to make sure that observations is at worst empty and never null
+        if (observations == null) {
+            observations = List.of();
+        }
+        this.observations = observations;
 
         // do this once so gets are quick
         getCameraIntrinsicsMat().get(0, 0, intrinsicsArr);
-        getDistCoeffsMat().get(0, 0, extrinsicsArr);
+        getDistCoeffsMat().get(0, 0, distCoeffsArr);
     }
 
     @JsonIgnore
@@ -82,18 +123,13 @@ public class CameraCalibrationCoefficients implements Releasable {
     }
 
     @JsonIgnore
-    public double[] getExtrinsicsArr() {
+    public double[] getDistCoeffsArr() {
-        return extrinsicsArr;
+        return distCoeffsArr;
     }
 
     @JsonIgnore
-    public double[] getPerViewErrors() {
+    public List<BoardObservation> getPerViewErrors() {
-        return perViewErrors;
+        return observations;
-    }
-
-    @JsonIgnore
-    public double getStandardDeviation() {
-        return standardDeviation;
     }
 
     @Override
@@ -130,14 +166,39 @@ public class CameraCalibrationCoefficients implements Releasable {
                     dist_coefs.get(4).doubleValue(),
                 };
 
-        var cam_jsonmat = new JsonMat(3, 3, cam_arr);
+        var cam_jsonmat = new JsonMatOfDouble(3, 3, cam_arr);
-        var distortion_jsonmat = new JsonMat(1, 5, dist_array);
+        var distortion_jsonmat = new JsonMatOfDouble(1, 5, dist_array);
 
-        var error = json.get("avg_reprojection_error").asDouble();
         var width = json.get("img_size").get(0).doubleValue();
         var height = json.get("img_size").get(1).doubleValue();
 
         return new CameraCalibrationCoefficients(
-                new Size(width, height), cam_jsonmat, distortion_jsonmat, new double[] {error}, 0);
+                new Size(width, height),
+                cam_jsonmat,
+                distortion_jsonmat,
+                new double[0],
+                List.of(),
+                new Size(0, 0),
+                0,
+                CameraLensModel.LENSMODEL_OPENCV);
+    }
+
+    @Override
+    public String toString() {
+        return "CameraCalibrationCoefficients [resolution="
+                + resolution
+                + ", cameraIntrinsics="
+                + cameraIntrinsics
+                + ", distCoeffs="
+                + distCoeffs
+                + ", observations="
+                + observations
+                + ", calobjectWarp="
+                + Arrays.toString(calobjectWarp)
+                + ", intrinsicsArr="
+                + Arrays.toString(intrinsicsArr)
+                + ", distCoeffsArr="
+                + Arrays.toString(distCoeffsArr)
+                + "]";
     }
 }

photon-core/src/main/java/org/photonvision/vision/calibration/CameraLensModel.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.calibration;
+
+/**
+ * What kind of camera lens model our intrinsics are modeling. For more info see:
+ * https://docs.opencv.org/4.x/dc/dbb/tutorial_py_calibration.html
+ * https://mrcal.secretsauce.net/lensmodels.html#org4e95788
+ */
+public enum CameraLensModel {
+    /** OpenCV[4,5,8,12]-based model */
+    LENSMODEL_OPENCV,
+    /** Mrcal steriographic lens model. See LENSMODEL_STEREOGRAPHIC in the mrcal docs */
+    LENSMODEL_STERIOGRAPHIC,
+    /**
+     * Mrcal splined-steriographic lens model. See LENSMODEL_SPLINED_STEREOGRAPHIC_ in the mrcal docs
+     */
+    LENSMODEL_SPLINED_STERIOGRAPHIC
+}

photon-core/src/main/java/org/photonvision/vision/calibration/JsonImageMat.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.calibration;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Base64;
+import org.opencv.core.Mat;
+import org.opencv.core.MatOfByte;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.photonvision.vision.opencv.Releasable;
+
+/** JSON-serializable image. Data is stored as base64-encoded PNG data. */
+public class JsonImageMat implements Releasable {
+    public final int rows;
+    public final int cols;
+    public final int type;
+
+    // We store image data as a base64-encoded PNG inside a Java string. This lets us serialize it
+    // without too much overhead and still use JSON.
+    public final String data;
+
+    // Cached matrices to avoid object recreation
+    @JsonIgnore private Mat wrappedMat = null;
+
+    public JsonImageMat(Mat mat) {
+        this.rows = mat.rows();
+        this.cols = mat.cols();
+        this.type = mat.type();
+
+        // Convert from Mat -> png byte array -> base64
+        var buf = new MatOfByte();
+        Imgcodecs.imencode(".png", mat, buf);
+        data = Base64.getEncoder().encodeToString(buf.toArray());
+        buf.release();
+    }
+
+    public JsonImageMat(
+            @JsonProperty("rows") int rows,
+            @JsonProperty("cols") int cols,
+            @JsonProperty("type") int type,
+            @JsonProperty("data") String data) {
+        this.rows = rows;
+        this.cols = cols;
+        this.type = type;
+        this.data = data;
+    }
+
+    @JsonIgnore
+    public Mat getAsMat() {
+        if (wrappedMat == null) {
+            // Convert back from base64 string -> png -> Mat
+            var bytes = Base64.getDecoder().decode(data);
+            var pngData = new MatOfByte(bytes);
+            this.wrappedMat = Imgcodecs.imdecode(pngData, Imgcodecs.IMREAD_COLOR);
+        }
+        return this.wrappedMat;
+    }
+
+    @Override
+    public void release() {
+        if (wrappedMat != null) wrappedMat.release();
+    }
+}

photon-core/src/main/java/org/photonvision/vision/calibration/JsonMatOfDouble.java

@@ -29,7 +29,8 @@ import org.opencv.core.MatOfDouble;
 import org.photonvision.common.dataflow.structures.Packet;
 import org.photonvision.vision.opencv.Releasable;
 
-public class JsonMat implements Releasable {
+/** JSON-serializable image. Data is stored as a raw JSON array. */
+public class JsonMatOfDouble implements Releasable {
     public final int rows;
     public final int cols;
     public final int type;
@@ -41,11 +42,11 @@ public class JsonMat implements Releasable {
 
     private MatOfDouble wrappedMatOfDouble;
 
-    public JsonMat(int rows, int cols, double[] data) {
+    public JsonMatOfDouble(int rows, int cols, double[] data) {
         this(rows, cols, CvType.CV_64FC1, data);
     }
 
-    public JsonMat(
+    public JsonMatOfDouble(
             @JsonProperty("rows") int rows,
             @JsonProperty("cols") int cols,
             @JsonProperty("type") int type,
@@ -84,9 +85,9 @@ public class JsonMat implements Releasable {
         return Arrays.copyOfRange(data, 0, dataLen);
     }
 
-    public static JsonMat fromMat(Mat mat) {
+    public static JsonMatOfDouble fromMat(Mat mat) {
         if (!isCalibrationMat(mat)) return null;
-        return new JsonMat(mat.rows(), mat.cols(), getDataFromMat(mat));
+        return new JsonMatOfDouble(mat.rows(), mat.cols(), getDataFromMat(mat));
     }
 
     @JsonIgnore
@@ -126,4 +127,23 @@ public class JsonMat implements Releasable {
         packet.encode(this.data);
         return packet;
     }
+
+    @Override
+    public String toString() {
+        return "JsonMat [rows="
+                + rows
+                + ", cols="
+                + cols
+                + ", type="
+                + type
+                + ", data="
+                + Arrays.toString(data)
+                + ", wrappedMat="
+                + wrappedMat
+                + ", wpilibMat="
+                + wpilibMat
+                + ", wrappedMatOfDouble="
+                + wrappedMatOfDouble
+                + "]";
+    }
 }

photon-core/src/main/java/org/photonvision/vision/camera/CameraInfo.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.camera;
+
+import edu.wpi.first.cscore.UsbCameraInfo;
+import java.util.Arrays;
+
+public class CameraInfo extends UsbCameraInfo {
+    public final CameraType cameraType;
+
+    public CameraInfo(
+            int dev, String path, String name, String[] otherPaths, int vendorId, int productId) {
+        super(dev, path, name, otherPaths, vendorId, productId);
+        cameraType = CameraType.UsbCamera;
+    }
+
+    public CameraInfo(
+            int dev,
+            String path,
+            String name,
+            String[] otherPaths,
+            int vendorId,
+            int productId,
+            CameraType cameraType) {
+        super(dev, path, name, otherPaths, vendorId, productId);
+        this.cameraType = cameraType;
+    }
+
+    public CameraInfo(UsbCameraInfo info) {
+        super(info.dev, info.path, info.name, info.otherPaths, info.vendorId, info.productId);
+        cameraType = CameraType.UsbCamera;
+    }
+
+    /**
+     * @return True, if this camera is reported from V4L and is a CSI camera.
+     */
+    public boolean getIsV4lCsiCamera() {
+        return (Arrays.stream(otherPaths).anyMatch(it -> it.contains("csi-video"))
+                || getBaseName().equals("unicam"));
+    }
+
+    /**
+     * @return The base name of the camera aka the name as just ascii.
+     */
+    public String getBaseName() {
+        return name.replaceAll("[^\\x00-\\x7F]", "");
+    }
+
+    /**
+     * @return Returns a human readable name
+     */
+    public String getHumanReadableName() {
+        return getBaseName().replaceAll(" ", "_");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (!(o instanceof UsbCameraInfo || o instanceof CameraInfo)) return false;
+        UsbCameraInfo other = (UsbCameraInfo) o;
+        return path.equals(other.path)
+                // && a.dev == b.dev (dev is not constant in Windows)
+                && name.equals(other.name)
+                && productId == other.productId
+                && vendorId == other.vendorId;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java

@@ -32,4 +32,13 @@ public enum CameraQuirk {
     AdjustableFocus,
     /** Changing FPS repeatedly with small delay does not work correctly */
     StickyFPS,
+    /** Camera is an arducam. This means it shares VID/PID with other arducams (ew) */
+    ArduCamCamera,
+    /**
+     * Camera is an arducam ov9281 which has a funky exposure issue where it is defined in v4l as
+     * 1-5000 instead of 1-75
+     */
+    ArduOV9281,
+    /** Dummy quirk to tell OV2311 from OV9281 */
+    ArduOV2311,
 }

photon-core/src/main/java/org/photonvision/vision/camera/FileVisionSource.java

@@ -18,7 +18,7 @@
 package org.photonvision.vision.camera;
 
 import edu.wpi.first.cscore.VideoMode;
-import edu.wpi.first.cscore.VideoMode.PixelFormat;
+import edu.wpi.first.util.PixelFormat;
 import java.nio.file.Path;
 import java.util.HashMap;
 import org.photonvision.common.configuration.CameraConfiguration;

photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSettables.java

@@ -18,7 +18,9 @@
 package org.photonvision.vision.camera;
 
 import edu.wpi.first.cscore.VideoMode;
+import edu.wpi.first.math.MathUtil;
 import edu.wpi.first.math.Pair;
+import edu.wpi.first.util.PixelFormat;
 import java.util.HashMap;
 import org.photonvision.common.configuration.CameraConfiguration;
 import org.photonvision.common.util.math.MathUtils;
@@ -34,11 +36,13 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
     private boolean lastAutoExposureActive;
     private int lastGain = 50;
     private Pair<Integer, Integer> lastAwbGains = new Pair<>(18, 18);
-    private boolean m_initialized = false;
+    public long r_ptr = 0;
 
     private final LibCameraJNI.SensorModel sensorModel;
 
-    private ImageRotationMode m_rotationMode;
+    private ImageRotationMode m_rotationMode = ImageRotationMode.DEG_0;
+
+    public final Object CAMERA_LOCK = new Object();
 
     public void setRotation(ImageRotationMode rotationMode) {
         if (rotationMode != m_rotationMode) {
@@ -53,56 +57,44 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
 
         videoModes = new HashMap<>();
 
-        sensorModel = LibCameraJNI.getSensorModel();
+        sensorModel = LibCameraJNI.getSensorModel(configuration.path);
 
         if (sensorModel == LibCameraJNI.SensorModel.IMX219) {
             // Settings for the IMX219 sensor, which is used on the Pi Camera Module v2
-            videoModes.put(
+            videoModes.put(0, new FPSRatedVideoMode(PixelFormat.kUnknown, 320, 240, 120, 120, .39));
-                    0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 120, 120, .39));
+            videoModes.put(1, new FPSRatedVideoMode(PixelFormat.kUnknown, 320, 240, 30, 30, .39));
-            videoModes.put(
+            videoModes.put(2, new FPSRatedVideoMode(PixelFormat.kUnknown, 640, 480, 65, 90, .39));
-                    1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39));
+            videoModes.put(3, new FPSRatedVideoMode(PixelFormat.kUnknown, 640, 480, 30, 30, .39));
-            videoModes.put(
-                    2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
-            videoModes.put(
-                    3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 30, 30, .39));
             // TODO: fix 1280x720 in the native code and re-add it
-            videoModes.put(
+            videoModes.put(4, new FPSRatedVideoMode(PixelFormat.kUnknown, 1920, 1080, 15, 20, .53));
-                    4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, .53));
+            videoModes.put(5, new FPSRatedVideoMode(PixelFormat.kUnknown, 3280 / 2, 2464 / 2, 15, 20, 1));
-            videoModes.put(
+            videoModes.put(6, new FPSRatedVideoMode(PixelFormat.kUnknown, 3280 / 4, 2464 / 4, 15, 20, 1));
-                    5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 2, 2464 / 2, 15, 20, 1));
-            videoModes.put(
-                    6, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 3280 / 4, 2464 / 4, 15, 20, 1));
         } else if (sensorModel == LibCameraJNI.SensorModel.OV9281) {
-            videoModes.put(
+            videoModes.put(0, new FPSRatedVideoMode(PixelFormat.kUnknown, 320, 240, 30, 30, .39));
-                    0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 30, 30, .39));
+            videoModes.put(1, new FPSRatedVideoMode(PixelFormat.kUnknown, 1280 / 2, 800 / 2, 60, 60, 1));
-            videoModes.put(
+            videoModes.put(2, new FPSRatedVideoMode(PixelFormat.kUnknown, 640, 480, 65, 90, .39));
-                    1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280 / 2, 800 / 2, 60, 60, 1));
+            videoModes.put(3, new FPSRatedVideoMode(PixelFormat.kUnknown, 1280, 800, 60, 60, 1));
-            videoModes.put(
-                    2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 65, 90, .39));
-            videoModes.put(
-                    3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 800, 60, 60, 1));
 
         } else {
             if (sensorModel == LibCameraJNI.SensorModel.IMX477) {
                 LibcameraGpuSource.logger.warn(
                         "It appears you are using a Pi HQ Camera. This camera is not officially supported. You will have to set your camera FOV differently based on resolution.");
+            } else if (sensorModel == LibCameraJNI.SensorModel.IMX708) {
+                LibcameraGpuSource.logger.warn(
+                        "It appears you are using a Pi Camera V3. This camera is not officially supported. You will have to set your camera FOV differently based on resolution.");
             } else if (sensorModel == LibCameraJNI.SensorModel.Unknown) {
                 LibcameraGpuSource.logger.warn(
                         "You have an unknown sensor connected to your Pi over CSI! This is likely a bug. If it is not, then you will have to set your camera FOV differently based on resolution.");
             }
 
             // Settings for the OV5647 sensor, which is used by the Pi Camera Module v1
-            videoModes.put(0, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 320, 240, 90, 90, 1));
+            videoModes.put(0, new FPSRatedVideoMode(PixelFormat.kUnknown, 320, 240, 90, 90, 1));
-            videoModes.put(1, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 640, 480, 85, 90, 1));
+            videoModes.put(1, new FPSRatedVideoMode(PixelFormat.kUnknown, 640, 480, 85, 90, 1));
-            videoModes.put(
+            videoModes.put(2, new FPSRatedVideoMode(PixelFormat.kUnknown, 960, 720, 45, 49, 0.74));
-                    2, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 960, 720, 45, 49, 0.74));
             // Half the size of the active areas on the OV5647
-            videoModes.put(
+            videoModes.put(3, new FPSRatedVideoMode(PixelFormat.kUnknown, 2592 / 2, 1944 / 2, 20, 20, 1));
-                    3, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 2592 / 2, 1944 / 2, 20, 20, 1));
+            videoModes.put(4, new FPSRatedVideoMode(PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91));
-            videoModes.put(
+            videoModes.put(5, new FPSRatedVideoMode(PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72));
-                    4, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1280, 720, 30, 45, 0.91));
-            videoModes.put(
-                    5, new FPSRatedVideoMode(VideoMode.PixelFormat.kUnknown, 1920, 1080, 15, 20, 0.72));
         }
 
         // TODO need to add more video modes for new sensors here
@@ -118,7 +110,7 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
     @Override
     public void setAutoExposure(boolean cameraAutoExposure) {
         lastAutoExposureActive = cameraAutoExposure;
-        LibCameraJNI.setAutoExposure(cameraAutoExposure);
+        LibCameraJNI.setAutoExposure(r_ptr, cameraAutoExposure);
     }
 
     @Override
@@ -128,23 +120,28 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
             return;
         }
 
+        // Store the exposure for use when we need to recreate the camera.
+        lastManualExposure = exposure;
+
+        // Minimum exposure can't be below 1uS cause otherwise it would be 0 and 0 is auto exposure.
+        double minExposure = 1;
+
         // HACKS!
         // If we set exposure too low, libcamera crashes or slows down
         // Very weird and smelly
         // For now, band-aid this by just not setting it lower than the "it breaks" limit
-        //  is different depending on camera.
+        // is different depending on camera.
+        // All units are uS.
         if (sensorModel == LibCameraJNI.SensorModel.OV9281) {
-            if (exposure < 6.0) {
+            minExposure = 4800;
-                exposure = 6.0;
-            }
         } else if (sensorModel == LibCameraJNI.SensorModel.OV5647) {
-            if (exposure < 0.7) {
+            minExposure = 560;
-                exposure = 0.7;
-            }
         }
+        // 80,000 uS seems like an exposure value that will be greater than ever needed while giving
+        // enough control over exposure.
+        exposure = MathUtils.map(exposure, 0, 100, minExposure, 80000);
 
-        lastManualExposure = exposure;
+        var success = LibCameraJNI.setExposure(r_ptr, (int) exposure);
-        var success = LibCameraJNI.setExposure((int) Math.round(exposure) * 800);
         if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera exposure");
     }
 
@@ -152,15 +149,19 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
     public void setBrightness(int brightness) {
         lastBrightness = brightness;
         double realBrightness = MathUtils.map(brightness, 0.0, 100.0, -1.0, 1.0);
-        var success = LibCameraJNI.setBrightness(realBrightness);
+        var success = LibCameraJNI.setBrightness(r_ptr, realBrightness);
         if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera brightness");
     }
 
     @Override
     public void setGain(int gain) {
         lastGain = gain;
-        // TODO units here seem odd -- 5ish seems legit? So divide by 10
+
-        var success = LibCameraJNI.setAnalogGain(gain / 10.0);
+        // Map and clamp gain to values between 1 and 10 (libcamera min and gain that just seems higher
+        // than ever needed) from 0 to 100 (UI values).
+        var success =
+                LibCameraJNI.setAnalogGain(
+                        r_ptr, MathUtil.clamp(MathUtils.map(gain, 0.0, 100.0, 1.0, 10.0), 1.0, 10.0));
         if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera gain");
     }
 
@@ -182,7 +183,7 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
 
     public void setAwbGain(int red, int blue) {
         if (sensorModel != LibCameraJNI.SensorModel.OV9281) {
-            var success = LibCameraJNI.setAwbGain(red / 10.0, blue / 10.0);
+            var success = LibCameraJNI.setAwbGain(r_ptr, red / 10.0, blue / 10.0);
             if (!success) LibcameraGpuSource.logger.warn("Couldn't set Pi Camera AWB gains");
         }
     }
@@ -198,29 +199,35 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
 
         // We need to make sure that other threads don't try to do anything funny while we're recreating
         // the camera
-        synchronized (LibCameraJNI.CAMERA_LOCK) {
+        synchronized (CAMERA_LOCK) {
-            if (m_initialized) {
+            if (r_ptr != 0) {
                 logger.debug("Stopping libcamera");
-                if (!LibCameraJNI.stopCamera()) {
+                if (!LibCameraJNI.stopCamera(r_ptr)) {
                     logger.error("Couldn't stop a zero copy Pi Camera while switching video modes");
                 }
                 logger.debug("Destroying libcamera");
-                if (!LibCameraJNI.destroyCamera()) {
+                if (!LibCameraJNI.destroyCamera(r_ptr)) {
                     logger.error("Couldn't destroy a zero copy Pi Camera while switching video modes");
                 }
             }
 
             logger.debug("Creating libcamera");
-            if (!LibCameraJNI.createCamera(
+            r_ptr =
-                    mode.width, mode.height, (m_rotationMode == ImageRotationMode.DEG_180 ? 180 : 0))) {
+                    LibCameraJNI.createCamera(
+                            getConfiguration().path,
+                            mode.width,
+                            mode.height,
+                            (m_rotationMode == ImageRotationMode.DEG_180 ? 180 : 0));
+            if (r_ptr == 0) {
                 logger.error("Couldn't create a zero copy Pi Camera while switching video modes");
+                if (!LibCameraJNI.destroyCamera(r_ptr)) {
+                    logger.error("Couldn't destroy a zero copy Pi Camera while switching video modes");
+                }
             }
             logger.debug("Starting libcamera");
-            if (!LibCameraJNI.startCamera()) {
+            if (!LibCameraJNI.startCamera(r_ptr)) {
                 logger.error("Couldn't start a zero copy Pi Camera while switching video modes");
             }
-
-            m_initialized = true;
         }
 
         // We don't store last settings on the native side, and when you change video mode these get
@@ -231,7 +238,7 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
         setGain(lastGain);
         setAwbGain(lastAwbGains.getFirst(), lastAwbGains.getSecond());
 
-        LibCameraJNI.setFramesToCopy(true, true);
+        LibCameraJNI.setFramesToCopy(r_ptr, true, true);
 
         currentVideoMode = mode;
     }
@@ -240,4 +247,8 @@ public class LibcameraGpuSettables extends VisionSourceSettables {
     public HashMap<Integer, VideoMode> getAllVideoModes() {
         return videoModes;
     }
+
+    public LibCameraJNI.SensorModel getModel() {
+        return sensorModel;
+    }
 }

photon-core/src/main/java/org/photonvision/vision/camera/LibcameraGpuSource.java

@@ -18,6 +18,7 @@
 package org.photonvision.vision.camera;
 
 import edu.wpi.first.cscore.VideoMode;
+import edu.wpi.first.util.PixelFormat;
 import org.photonvision.common.configuration.CameraConfiguration;
 import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.logging.LogGroup;

photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java

@@ -17,6 +17,10 @@
 
 package org.photonvision.vision.camera;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Objects;
@@ -45,8 +49,32 @@ public class QuirkyCamera {
                             -1, -1, "mmal service 16.1", CameraQuirk.PiCam), // PiCam (via V4L2, not zerocopy)
                     new QuirkyCamera(-1, -1, "unicam", CameraQuirk.PiCam), // PiCam (via V4L2, not zerocopy)
                     new QuirkyCamera(0x85B, 0x46D, CameraQuirk.AdjustableFocus), // Logitech C925-e
-                    new QuirkyCamera(0x6366, 0x0c45, CameraQuirk.StickyFPS) // Arducam OV2311
+                    // Generic arducam. Since OV2311 can't be differentiated at first boot, apply stickyFPS to
-                    );
+                    // the generic case, too
+                    new QuirkyCamera(
+                            0x0c45,
+                            0x6366,
+                            "",
+                            "Arducam Generic",
+                            CameraQuirk.ArduCamCamera,
+                            CameraQuirk.StickyFPS),
+                    // Arducam OV2311
+                    new QuirkyCamera(
+                            0x0c45,
+                            0x6366,
+                            "OV2311",
+                            "OV2311",
+                            CameraQuirk.ArduCamCamera,
+                            CameraQuirk.ArduOV2311,
+                            CameraQuirk.StickyFPS),
+                    // Arducam OV9281
+                    new QuirkyCamera(
+                            0x0c45,
+                            0x6366,
+                            "OV9281",
+                            "OV9281",
+                            CameraQuirk.ArduCamCamera,
+                            CameraQuirk.ArduOV9281));
 
     public static final QuirkyCamera DefaultCamera = new QuirkyCamera(0, 0, "");
     public static final QuirkyCamera ZeroCopyPiCamera =
@@ -58,9 +86,19 @@ public class QuirkyCamera {
                     CameraQuirk.Gain,
                     CameraQuirk.AWBGain); // PiCam (special zerocopy version)
 
+    @JsonProperty("baseName")
     public final String baseName;
+
+    @JsonProperty("usbVid")
     public final int usbVid;
+
+    @JsonProperty("usbPid")
     public final int usbPid;
+
+    @JsonProperty("displayName")
+    public final String displayName;
+
+    @JsonProperty("quirks")
     public final HashMap<CameraQuirk, Boolean> quirks;
 
     /**
@@ -83,9 +121,24 @@ public class QuirkyCamera {
      * @param quirks Camera quirks
      */
     private QuirkyCamera(int usbVid, int usbPid, String baseName, CameraQuirk... quirks) {
+        this(usbVid, usbPid, baseName, "", quirks);
+    }
+
+    /**
+     * Creates a QuirkyCamera that matches by USB VID/PID and name
+     *
+     * @param usbVid USB VID of camera
+     * @param usbPid USB PID of camera
+     * @param baseName CSCore name of camera
+     * @param displayName Human-friendly quicky camera name
+     * @param quirks Camera quirks
+     */
+    private QuirkyCamera(
+            int usbVid, int usbPid, String baseName, String displayName, CameraQuirk... quirks) {
         this.usbVid = usbVid;
         this.usbPid = usbPid;
         this.baseName = baseName;
+        this.displayName = displayName;
 
         this.quirks = new HashMap<>();
         for (var q : quirks) {
@@ -96,6 +149,20 @@ public class QuirkyCamera {
         }
     }
 
+    @JsonCreator
+    public QuirkyCamera(
+            @JsonProperty("baseName") String baseName,
+            @JsonProperty("usbVid") int usbVid,
+            @JsonProperty("usbPid") int usbPid,
+            @JsonProperty("displayName") String displayName,
+            @JsonProperty("quirks") HashMap<CameraQuirk, Boolean> quirks) {
+        this.baseName = baseName;
+        this.usbPid = usbPid;
+        this.usbVid = usbVid;
+        this.quirks = quirks;
+        this.displayName = displayName;
+    }
+
     public boolean hasQuirk(CameraQuirk quirk) {
         return quirks.get(quirk);
     }
@@ -108,8 +175,20 @@ public class QuirkyCamera {
         for (var qc : quirkyCameras) {
             boolean hasBaseName = !qc.baseName.isEmpty();
             boolean matchesBaseName = qc.baseName.equals(baseName) || !hasBaseName;
+            // If we have a quirkycamera we need to copy the quirks from our predefined object and create
+            // a quirkycamera object with the baseName.
             if (qc.usbVid == usbVid && qc.usbPid == usbPid && matchesBaseName) {
-                return qc;
+                List<CameraQuirk> quirks = new ArrayList<CameraQuirk>();
+                for (var q : CameraQuirk.values()) {
+                    if (qc.hasQuirk(q)) quirks.add(q);
+                }
+                QuirkyCamera c =
+                        new QuirkyCamera(
+                                usbVid,
+                                usbPid,
+                                baseName,
+                                Arrays.copyOf(quirks.toArray(), quirks.size(), CameraQuirk[].class));
+                return c;
             }
         }
         return new QuirkyCamera(usbVid, usbPid, baseName);
@@ -130,8 +209,39 @@ public class QuirkyCamera {
                 && Objects.equals(quirks, that.quirks);
     }
 
+    @Override
+    public String toString() {
+        String ret =
+                "QuirkyCamera [baseName="
+                        + baseName
+                        + ", displayName="
+                        + displayName
+                        + ", usbVid="
+                        + usbVid
+                        + ", usbPid="
+                        + usbPid
+                        + ", quirks="
+                        + quirks.toString()
+                        + "]";
+        return ret;
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(usbVid, usbPid, baseName, quirks);
     }
+
+    /**
+     * Add/remove quirks from the camera we're controlling
+     *
+     * @param quirksToChange map of true/false for quirks we should change
+     */
+    public void updateQuirks(HashMap<CameraQuirk, Boolean> quirksToChange) {
+        for (var q : quirksToChange.entrySet()) {
+            var quirk = q.getKey();
+            var hasQuirk = q.getValue();
+
+            this.quirks.put(quirk, hasQuirk);
+        }
+    }
 }

photon-core/src/main/java/org/photonvision/vision/camera/USBCameraSource.java

@@ -22,13 +22,18 @@ import edu.wpi.first.cscore.CvSink;
 import edu.wpi.first.cscore.UsbCamera;
 import edu.wpi.first.cscore.VideoException;
 import edu.wpi.first.cscore.VideoMode;
+import edu.wpi.first.cscore.VideoProperty.Kind;
+import edu.wpi.first.util.PixelFormat;
 import java.util.*;
 import java.util.stream.Collectors;
 import org.photonvision.common.configuration.CameraConfiguration;
 import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
+import org.photonvision.common.util.TestUtils;
+import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.vision.frame.FrameProvider;
+import org.photonvision.vision.frame.provider.FileFrameProvider;
 import org.photonvision.vision.frame.provider.USBFrameProvider;
 import org.photonvision.vision.processes.VisionSource;
 import org.photonvision.vision.processes.VisionSourceSettables;
@@ -37,11 +42,9 @@ public class USBCameraSource extends VisionSource {
     private final Logger logger;
     private final UsbCamera camera;
     private final USBCameraSettables usbCameraSettables;
-    private final USBFrameProvider usbFrameProvider;
+    private FrameProvider usbFrameProvider;
     private final CvSink cvSink;
 
-    public final QuirkyCamera cameraQuirks;
-
     public USBCameraSource(CameraConfiguration config) {
         super(config);
 
@@ -49,17 +52,21 @@ public class USBCameraSource extends VisionSource {
         camera = new UsbCamera(config.nickname, config.path);
         cvSink = CameraServer.getVideo(this.camera);
 
-        cameraQuirks =
+        if (getCameraConfiguration().cameraQuirks == null)
-                QuirkyCamera.getQuirkyCamera(
+            getCameraConfiguration().cameraQuirks =
-                        camera.getInfo().productId, camera.getInfo().vendorId, config.baseName);
+                    QuirkyCamera.getQuirkyCamera(
+                            camera.getInfo().vendorId, camera.getInfo().productId, config.baseName);
 
-        if (cameraQuirks.hasQuirks()) {
+        if (getCameraConfiguration().cameraQuirks.hasQuirks()) {
-            logger.info("Quirky camera detected: " + cameraQuirks.baseName);
+            logger.info("Quirky camera detected: " + getCameraConfiguration().cameraQuirks.baseName);
         }
 
-        if (cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken)) {
+        if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken)) {
             // set some defaults, as these should never be used.
-            logger.info("Camera " + cameraQuirks.baseName + " is not supported for PhotonVision");
+            logger.info(
+                    "Camera "
+                            + getCameraConfiguration().cameraQuirks.baseName
+                            + " is not supported for PhotonVision");
             usbCameraSettables = null;
             usbFrameProvider = null;
         } else {
@@ -77,8 +84,26 @@ public class USBCameraSource extends VisionSource {
         }
     }
 
+    /**
+     * Mostly just used for unit tests to better simulate a usb camera without a camera being present.
+     */
+    public USBCameraSource(CameraConfiguration config, int pid, int vid, boolean unitTest) {
+        this(config);
+
+        if (getCameraConfiguration().cameraQuirks == null)
+            getCameraConfiguration().cameraQuirks =
+                    QuirkyCamera.getQuirkyCamera(pid, vid, config.baseName);
+
+        if (unitTest)
+            usbFrameProvider =
+                    new FileFrameProvider(
+                            TestUtils.getWPIImagePath(
+                                    TestUtils.WPI2019Image.kCargoStraightDark72in_HighRes, false),
+                            TestUtils.WPI2019Image.FOV);
+    }
+
     void disableAutoFocus() {
-        if (cameraQuirks.hasQuirk(CameraQuirk.AdjustableFocus)) {
+        if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.AdjustableFocus)) {
             try {
                 camera.getProperty("focus_auto").set(0);
                 camera.getProperty("focus_absolute").set(0); // Focus into infinity
@@ -88,6 +113,10 @@ public class USBCameraSource extends VisionSource {
         }
     }
 
+    public QuirkyCamera getCameraQuirks() {
+        return getCameraConfiguration().cameraQuirks;
+    }
+
     @Override
     public FrameProvider getFrameProvider() {
         return usbFrameProvider;
@@ -99,17 +128,21 @@ public class USBCameraSource extends VisionSource {
     }
 
     public class USBCameraSettables extends VisionSourceSettables {
+        // We need to remember the last exposure set when exiting auto exposure mode so we can restore
+        // it
+        private double last_exposure = -1;
+
         protected USBCameraSettables(CameraConfiguration configuration) {
             super(configuration);
             getAllVideoModes();
-            if (!cameraQuirks.hasQuirk(CameraQuirk.StickyFPS))
+            if (!configuration.cameraQuirks.hasQuirk(CameraQuirk.StickyFPS))
-                setVideoMode(videoModes.get(0)); // fixes double FPS set
+                if (!videoModes.isEmpty()) setVideoMode(videoModes.get(0)); // fixes double FPS set
         }
 
         public void setAutoExposure(boolean cameraAutoExposure) {
             logger.debug("Setting auto exposure to " + cameraAutoExposure);
 
-            if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
+            if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
                 // Case, we know this is a picam. Go through v4l2-ctl interface directly
 
                 // Common settings
@@ -141,20 +174,46 @@ public class USBCameraSource extends VisionSource {
             } else {
                 // Case - this is some other USB cam. Default to wpilib's implementation
 
-                var canSetWhiteBalance = !cameraQuirks.hasQuirk(CameraQuirk.Gain);
+                var canSetWhiteBalance = !getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.Gain);
 
                 if (!cameraAutoExposure) {
                     // Pick a bunch of reasonable setting defaults for vision processing retroreflective
                     if (canSetWhiteBalance) {
-                        camera.setWhiteBalanceManual(4000); // Auto white-balance disabled, 4000K preset
+                        // Linux kernel bump changed names -- now called white_balance_automatic and
+                        // white_balance_temperature
+                        if (camera.getProperty("white_balance_automatic").getKind() != Kind.kNone) {
+                            // 1=auto, 0=manual
+                            camera.getProperty("white_balance_automatic").set(0);
+                            camera.getProperty("white_balance_temperature").set(4000);
+                        } else {
+                            camera.setWhiteBalanceManual(4000); // Auto white-balance disabled, 4000K preset
+                        }
+
+                        // Most cameras leave exposure time absolute at the last value from their AE algorithm.
+                        // Set it back to the exposure slider value
+                        setExposure(this.last_exposure);
                     }
                 } else {
                     // Pick a bunch of reasonable setting defaults for driver, fiducials, or otherwise
                     // nice-for-humans
                     if (canSetWhiteBalance) {
-                        camera.setWhiteBalanceAuto(); // Auto white-balance enabled
+                        // Linux kernel bump changed names -- now called white_balance_automatic
+                        if (camera.getProperty("white_balance_automatic").getKind() != Kind.kNone) {
+                            // 1=auto, 0=manual
+                            camera.getProperty("white_balance_automatic").set(1);
+                        } else {
+                            camera.setWhiteBalanceAuto(); // Auto white-balance enabled
+                        }
+                    }
+
+                    // Linux kernel bump changed names -- exposure_auto is now called auto_exposure
+                    if (camera.getProperty("auto_exposure").getKind() != Kind.kNone) {
+                        var prop = camera.getProperty("auto_exposure");
+                        // 3=auto-aperature
+                        prop.set((int) 3);
+                    } else {
+                        camera.setExposureAuto(); // auto exposure enabled
                     }
-                    camera.setExposureAuto(); // auto exposure enabled
                 }
             }
         }
@@ -182,12 +241,31 @@ public class USBCameraSource extends VisionSource {
             if (exposure >= 0.0) {
                 try {
                     int scaledExposure = 1;
-                    if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
+                    if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
                         scaledExposure = Math.round(timeToPiCamRawExposure(pctToExposureTimeUs(exposure)));
                         logger.debug("Setting camera raw exposure to " + scaledExposure);
                         camera.getProperty("raw_exposure_time_absolute").set(scaledExposure);
                         camera.getProperty("raw_exposure_time_absolute").set(scaledExposure);
 
+                        // Yay thanks v4l for changing names randomly
+                    } else if (camera.getProperty("exposure_time_absolute").getKind() != Kind.kNone
+                            && camera.getProperty("auto_exposure").getKind() != Kind.kNone) {
+                        // 1=manual-aperature
+                        camera.getProperty("auto_exposure").set(1);
+
+                        // Seems like the name changed at some point in v4l? set it ouyrselves too
+                        var prop = camera.getProperty("raw_exposure_time_absolute");
+
+                        var propMin = prop.getMin();
+                        var propMax = prop.getMax();
+
+                        if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.ArduOV9281)) {
+                            propMin = 1;
+                            propMax = 75;
+                        }
+
+                        var exposure_manual_val = MathUtils.map(Math.round(exposure), 0, 100, propMin, propMax);
+                        prop.set((int) exposure_manual_val);
                     } else {
                         scaledExposure = (int) Math.round(exposure);
                         logger.debug("Setting camera exposure to " + scaledExposure);
@@ -197,6 +275,7 @@ public class USBCameraSource extends VisionSource {
                 } catch (VideoException e) {
                     logger.error("Failed to set camera exposure!", e);
                 }
+                this.last_exposure = exposure;
             }
         }
 
@@ -213,7 +292,7 @@ public class USBCameraSource extends VisionSource {
         @Override
         public void setGain(int gain) {
             try {
-                if (cameraQuirks.hasQuirk(CameraQuirk.Gain)) {
+                if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.Gain)) {
                     camera.getProperty("gain_automatic").set(0);
                     camera.getProperty("gain").set(gain);
                 }
@@ -247,41 +326,41 @@ public class USBCameraSource extends VisionSource {
                 List<VideoMode> videoModesList = new ArrayList<>();
                 try {
                     VideoMode[] modes;
-                    if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
+                    if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
                         modes =
                                 new VideoMode[] {
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 90),
+                                    new VideoMode(PixelFormat.kBGR, 320, 240, 90),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 30),
+                                    new VideoMode(PixelFormat.kBGR, 320, 240, 30),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 15),
+                                    new VideoMode(PixelFormat.kBGR, 320, 240, 15),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 320, 240, 10),
+                                    new VideoMode(PixelFormat.kBGR, 320, 240, 10),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 90),
+                                    new VideoMode(PixelFormat.kBGR, 640, 480, 90),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 45),
+                                    new VideoMode(PixelFormat.kBGR, 640, 480, 45),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 30),
+                                    new VideoMode(PixelFormat.kBGR, 640, 480, 30),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 15),
+                                    new VideoMode(PixelFormat.kBGR, 640, 480, 15),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 640, 480, 10),
+                                    new VideoMode(PixelFormat.kBGR, 640, 480, 10),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 960, 720, 60),
+                                    new VideoMode(PixelFormat.kBGR, 960, 720, 60),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 960, 720, 10),
+                                    new VideoMode(PixelFormat.kBGR, 960, 720, 10),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 1280, 720, 45),
+                                    new VideoMode(PixelFormat.kBGR, 1280, 720, 45),
-                                    new VideoMode(VideoMode.PixelFormat.kBGR, 1920, 1080, 20),
+                                    new VideoMode(PixelFormat.kBGR, 1920, 1080, 20),
                                 };
                     } else {
                         modes = camera.enumerateVideoModes();
                     }
                     for (VideoMode videoMode : modes) {
                         // Filter grey modes
-                        if (videoMode.pixelFormat == VideoMode.PixelFormat.kGray
+                        if (videoMode.pixelFormat == PixelFormat.kGray
-                                || videoMode.pixelFormat == VideoMode.PixelFormat.kUnknown) {
+                                || videoMode.pixelFormat == PixelFormat.kUnknown) {
                             continue;
                         }
 
                         // On picam, filter non-bgr modes for performance
-                        if (cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
+                        if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.PiCam)) {
-                            if (videoMode.pixelFormat != VideoMode.PixelFormat.kBGR) {
+                            if (videoMode.pixelFormat != PixelFormat.kBGR) {
                                 continue;
                             }
                         }
 
-                        if (cameraQuirks.hasQuirk(CameraQuirk.FPSCap100)) {
+                        if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.FPSCap100)) {
                             if (videoMode.fps > 100) {
                                 continue;
                             }
@@ -339,20 +418,43 @@ public class USBCameraSource extends VisionSource {
     @Override
     public boolean isVendorCamera() {
         return ConfigManager.getInstance().getConfig().getHardwareConfig().hasPresetFOV()
-                && cameraQuirks.hasQuirk(CameraQuirk.PiCam);
+                && getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.PiCam);
     }
 
     @Override
-    public boolean equals(Object o) {
+    public boolean equals(Object obj) {
-        if (this == o) return true;
+        if (this == obj) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+        if (obj == null) return false;
-        USBCameraSource that = (USBCameraSource) o;
+        if (getClass() != obj.getClass()) return false;
-        return cameraQuirks.equals(that.cameraQuirks);
+        USBCameraSource other = (USBCameraSource) obj;
+        if (camera == null) {
+            if (other.camera != null) return false;
+        } else if (!camera.equals(other.camera)) return false;
+        if (usbCameraSettables == null) {
+            if (other.usbCameraSettables != null) return false;
+        } else if (!usbCameraSettables.equals(other.usbCameraSettables)) return false;
+        if (usbFrameProvider == null) {
+            if (other.usbFrameProvider != null) return false;
+        } else if (!usbFrameProvider.equals(other.usbFrameProvider)) return false;
+        if (cvSink == null) {
+            if (other.cvSink != null) return false;
+        } else if (!cvSink.equals(other.cvSink)) return false;
+        if (getCameraConfiguration().cameraQuirks == null) {
+            if (other.getCameraConfiguration().cameraQuirks != null) return false;
+        } else if (!getCameraConfiguration()
+                .cameraQuirks
+                .equals(other.getCameraConfiguration().cameraQuirks)) return false;
+        return true;
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(
-                camera, usbCameraSettables, usbFrameProvider, cameraConfiguration, cvSink, cameraQuirks);
+                camera,
+                usbCameraSettables,
+                usbFrameProvider,
+                cameraConfiguration,
+                cvSink,
+                getCameraConfiguration().cameraQuirks);
     }
 }