Run build

* Recreate user pipeline on type change

* Fix typo

---------

Co-authored-by: shueja <32416547+shueja@users.noreply.github.com>

.github/workflows/build.yml

@@ -2,123 +2,14 @@ name: Build
 
 on:
   push:
-    branches: [ master ]
+    branches: 
+      - master
     tags:
       - 'v*'
   pull_request:
     branches: [ master ]
 
 jobs:
-  build-client:
-    name: "PhotonClient Build"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Build Production Client
-        run: npm run build
-      - uses: actions/upload-artifact@v4
-        with:
-          name: built-client
-          path: photon-client/dist/
-  build-examples:
-    name: "Build Examples"
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      # Need to publish to maven local first, so that C++ sim can pick it up
-      # Still haven't figured out how to make the vendordep file be copied before trying to build examples
-      - name: Publish photonlib to maven local
-        run: |
-          chmod +x gradlew
-          ./gradlew publishtomavenlocal -x check
-      - name: Build Java examples
-        working-directory: photonlib-java-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-      - name: Build C++ examples
-        working-directory: photonlib-cpp-examples
-        run: |
-          chmod +x gradlew
-          ./gradlew copyPhotonlib -x check
-          ./gradlew build -x check --max-workers 2
-  build-gradle:
-    name: "Gradle Build"
-    runs-on: ubuntu-22.04
-    steps:
-      # Checkout code.
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Fetch tags
-        run: git fetch --tags --force
-      - name: Install Java 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Install mrcal deps
-        run: sudo apt-get update && sudo apt-get install -y libcholmod3 liblapack3 libsuitesparseconfig5
-      - name: Gradle Build
-        run: |
-          chmod +x gradlew
-          ./gradlew build -x check --max-workers 2
-      - name: Gradle Tests
-        run: ./gradlew testHeadless -i --max-workers 1 --stacktrace
-      - name: Gradle Coverage
-        run: ./gradlew jacocoTestReport --max-workers 1
-      - name: Publish Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-server/build/reports/jacoco/test/jacocoTestReport.xml
-      - name: Publish Core Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./photon-core/build/reports/jacoco/test/jacocoTestReport.xml
-  build-offline-docs:
-    name: "Build Offline Docs"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: 'PhotonVision/photonvision-docs.git'
-          ref: master
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.9'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install sphinx sphinx_rtd_theme sphinx-tabs sphinxext-opengraph doc8
-          pip install -r requirements.txt
-      - name: Build the docs
-        run: |
-          make html
-      - uses: actions/upload-artifact@master
-        with:
-          name: built-docs
-          path: build/html
   build-photonlib-host:
     env:
       MACOSX_DEPLOYMENT_TARGET: 12
@@ -188,179 +79,3 @@ jobs:
         env:
           ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }}
         if: github.event_name == 'push'
-  build-package:
-    needs: [build-client, build-gradle, build-offline-docs]
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: windows-latest
-            artifact-name: Win64
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOS
-            architecture: x64
-            arch-override: none
-          - os: ubuntu-latest
-            artifact-name: Linux
-            architecture: x64
-            arch-override: none
-          - os: macos-latest
-            artifact-name: macOSArm
-            architecture: x64
-            arch-override: macarm64
-          - os: ubuntu-latest
-            artifact-name: LinuxArm32
-            architecture: x64
-            arch-override: linuxarm32
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            architecture: x64
-            arch-override: linuxarm64
-
-    runs-on: ${{ matrix.os }}
-    name: "Build fat JAR - ${{ matrix.artifact-name }}"
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install Java 17
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          rm -rf photon-server/src/main/resources/web/*
-          mkdir -p photon-server/src/main/resources/web/docs
-        if: ${{ (matrix.os) != 'windows-latest' }}
-      - run: |
-          del photon-server\src\main\resources\web\*.*
-          mkdir photon-server\src\main\resources\web\docs
-        if: ${{ (matrix.os) == 'windows-latest' }}
-      - uses: actions/download-artifact@v4
-        with:
-          name: built-client
-          path: photon-server/src/main/resources/web/
-      - uses: actions/download-artifact@v4
-        with:
-          name: built-docs
-          path: photon-server/src/main/resources/web/docs
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2 -PArchOverride=${{ matrix.arch-override }}
-        if: ${{ (matrix.arch-override != 'none') }}
-      - run: |
-          chmod +x gradlew
-          ./gradlew photon-server:shadowJar --max-workers 2
-        if: ${{ (matrix.arch-override == 'none') }}
-      - uses: actions/upload-artifact@v4
-        with:
-          name: jar-${{ matrix.artifact-name }}
-          path: photon-server/build/libs
-  build-image:
-    needs: [build-package]
-
-    if: ${{ github.event_name != 'pull_request' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: RaspberryPi
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_raspi.img.xz
-            cpu: cortex-a7
-            image_additional_mb: 0
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: limelight2
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_limelight.img.xz
-            cpu: cortex-a7
-            image_additional_mb: 0
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: limelight3
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.5/photonvision_limelight3.img.xz
-            cpu: cortex-a7
-            image_additional_mb: 0
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: orangepi5
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.9/photonvision_opi5.img.xz
-            cpu: cortex-a8
-            image_additional_mb: 4096
-          - os: ubuntu-latest
-            artifact-name: LinuxArm64
-            image_suffix: orangepi5plus
-            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.9/photonvision_opi5plus.img.xz
-            cpu: cortex-a8
-            image_additional_mb: 4096
-
-    runs-on: ${{ matrix.os }}
-    name: "Build image - ${{ matrix.image_url }}"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: actions/download-artifact@v4
-        with:
-          name: jar-${{ matrix.artifact-name }}
-      - uses: pguyot/arm-runner-action@v2
-        name: Generate image
-        id: generate_image
-        with:
-          base_image: ${{ matrix.image_url }}
-          image_additional_mb: ${{ matrix.image_additional_mb }}
-          optimize_image: yes
-          cpu: ${{ matrix.cpu }}
-          # We do _not_ wanna copy photon into the image. Bind mount instead
-          bind_mount_repository: true
-          commands: |
-            chmod +x scripts/armrunner.sh
-            ./scripts/armrunner.sh
-      - name: Compress image
-        run: |
-          new_jar=$(realpath $(find . -name photonvision\*-linuxarm64.jar))
-          new_image_name=$(basename "${new_jar/.jar/_${{ matrix.image_suffix }}.img}")
-          mv ${{ steps.generate_image.outputs.image }} $new_image_name
-          sudo xz -T 0 -v $new_image_name
-      - uses: actions/upload-artifact@v4
-        name: Upload image
-        with:
-          name: image-${{ matrix.image_suffix }}
-          path: photonvision*.xz
-  release:
-    needs: [build-package, build-image]
-    runs-on: ubuntu-22.04
-    steps:
-      # Download literally every single artifact. This also downloads client and docs,
-      # but the filtering below won't pick these up (I hope)
-      - uses: actions/download-artifact@v4
-      - run: find
-      # Push to dev release
-      - uses: pyTooling/Actions/releaser@r0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          tag: 'Dev'
-          rm: true
-          files: |
-            **/*.xz
-            **/*.jar
-            **/photonlib*.json
-        if: github.event_name == 'push'
-      # Upload all jars and xz archives
-      - uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            **/*.xz
-            **/*.jar
-            **/photonlib*.json
-        if: startsWith(github.ref, 'refs/tags/v')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/documentation.yml

@@ -1,86 +0,0 @@
-name: Documentation
-
-on:
-  push:
-    # For now, run on all commits to master
-    branches: [ master ]
-    # and also all tags starting with v
-    tags:
-      - 'v*'
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-client:
-    name: "PhotonClient Build"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Build Production Client
-        run: npm run build-demo
-      - uses: actions/upload-artifact@v4
-        with:
-          name: built-client
-          path: photon-client/dist/
-
-  run_docs:
-    runs-on: "ubuntu-22.04"
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Fetch tags
-      run: git fetch --tags --force
-    - name: Install Java 17
-      uses: actions/setup-java@v3
-      with:
-        java-version: 17
-        distribution: temurin
-
-    - name: Build javadocs/doxygen
-      run: |
-        chmod +x gradlew
-        ./gradlew docs:generateJavaDocs docs:doxygen
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: built-docs
-        path: docs/build/docs
-
-  release:
-    needs: [build-client, run_docs]
-
-    runs-on: ubuntu-22.04
-    steps:
-
-      # Download literally every single artifact.
-      - uses: actions/download-artifact@v4
-
-      - run: find .
-      - name: copy file via ssh password
-        uses: appleboy/scp-action@v0.1.7
-        with:
-          host: ${{ secrets.WEBMASTER_SSH_HOST }}
-          username: ${{ secrets.WEBMASTER_SSH_USERNAME }}
-          password: ${{ secrets.WEBMASTER_SSH_KEY }}
-          port: ${{ secrets.WEBMASTER_SSH_PORT }}
-          source: "*"
-          target: /var/www/html/photonvision-docs/

.github/workflows/lint-format.yml

@@ -1,88 +0,0 @@
-name: Lint and Format
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
-  pull_request:
-    branches: [ master ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  wpiformat:
-    name: "wpiformat"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Install wpiformat
-        run: pip3 install wpiformat
-      - name: Run
-        run: wpiformat
-      - name: Check output
-        run: git --no-pager diff --exit-code HEAD
-      - name: Generate diff
-        run: git diff HEAD > wpiformat-fixes.patch
-        if: ${{ failure() }}
-      - uses: actions/upload-artifact@v3
-        with:
-          name: wpiformat fixes
-          path: wpiformat-fixes.patch
-        if: ${{ failure() }}
-  javaformat:
-    name: "Java Formatting"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-      - run: |
-          chmod +x gradlew
-          ./gradlew spotlessCheck
-
-  client-lint-format:
-    name: "PhotonClient Lint and Formatting"
-    defaults:
-      run:
-        working-directory: photon-client
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - name: Install Dependencies
-        run: npm ci
-      - name: Check Linting
-        run: npm run lint-ci
-      - name: Check Formatting
-        run: npm run format-ci
-  server-index:
-    name: "Check server index.html not changed"
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Fetch all history and metadata
-        run: |
-          git fetch --prune --unshallow
-          git checkout -b pr
-          git branch -f master origin/master
-      - name: Check index.html not changed
-        run: git --no-pager diff --exit-code origin/master photon-server/src/main/resources/web/index.html

.github/workflows/python.yml

@@ -1,60 +0,0 @@
-name: Build and Distribute PhotonLibPy
-
-permissions:
-  id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - 'v*'
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  buildAndDeploy:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.11
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel pytest
-
-      - name: Build wheel
-        working-directory: ./photon-lib/py
-        run: |
-          python setup.py sdist bdist_wheel
-
-      - name: Run Unit Tests
-        working-directory: ./photon-lib/py
-        run: |
-            pip install --no-cache-dir dist/*.whl
-            pytest
-
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@master
-        with:
-          name: dist
-          path: ./photon-lib/py/dist/
-
-      - name: Publish package distributions to TestPyPI
-        # Only upload on tags
-        if: startsWith(github.ref, 'refs/tags/v')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages_dir: ./photon-lib/py/dist/
-
-    permissions:
-      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing

build.gradle

@@ -4,7 +4,7 @@ plugins {
     id "com.diffplug.spotless" version "6.24.0"
     id "edu.wpi.first.NativeUtils" version "2024.6.1" apply false
     id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
     id 'edu.wpi.first.WpilibTools' version '1.3.0'
     id 'com.google.protobuf' version '0.9.4' apply false
 }
@@ -13,8 +13,9 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
-        maven { url = "https://maven.photonvision.org/repository/snapshots/" }
+        maven { url = "https://maven.photonvision.org/releases" }
+        maven { url = "https://maven.photonvision.org/snapshots" }
+        maven { url = "https://jogamp.org/deployment/maven/" }
     }
     wpilibRepositories.addAllReleaseRepositories(it)
     wpilibRepositories.addAllDevelopmentRepositories(it)
@@ -24,15 +25,15 @@ allprojects {
 apply from: "versioningHelper.gradle"
 
 ext {
-    wpilibVersion = "2024.3.1"
+    wpilibVersion = "2024.3.2"
     wpimathVersion = wpilibVersion
     openCVversion = "4.8.0-2"
-    joglVersion = "2.4.0-rc-20200307"
+    joglVersion = "2.4.0"
     javalinVersion = "5.6.2"
-    photonGlDriverLibVersion = "dev-v2023.1.0-9-g75fc678"
-    rknnVersion = "dev-v2024.0.0-64-gc0836a6"
+    photonGlDriverLibVersion = "dev-v2023.1.0-11-g2b7036f"
+    rknnVersion = "dev-v2024.0.1-4-g0db16ac"
     frcYear = "2024"
-    mrcalVersion = "dev-v2024.0.0-7-gc976aaa";
+    mrcalVersion = "dev-v2024.0.0-24-gc1efcf0";
 
 
     pubVersion = versionString
@@ -50,6 +51,10 @@ ext {
     println("Building for platform " + jniPlatform + " wpilib: " + wpilibNativeName)
     println("Using Wpilib: " + wpilibVersion)
     println("Using OpenCV: " + openCVversion)
+
+
+    photonMavenURL = 'https://maven.photonvision.org/' + (isDev ? 'snapshots' : 'releases');
+    println("Publishing Photonlib to " + photonMavenURL)
 }
 
 spotless {

devTools/calibrationUtils.py

@@ -162,9 +162,9 @@ def __convert_cal_to_mrcal_cameramodel(
         "indices_point_camintrinsics_camextrinsics": None,
         "lensmodel": model,
         "imagersizes": np.array([imagersize], dtype=np.int32),
-        "calobject_warp": np.array(cal.calobjectWarp)
-        if len(cal.calobjectWarp) > 0
-        else None,
+        "calobject_warp": (
+            np.array(cal.calobjectWarp) if len(cal.calobjectWarp) > 0 else None
+        ),
         # We always do all the things
         "do_optimize_intrinsics_core": True,
         "do_optimize_intrinsics_distortions": True,

photon-client/src/components/dashboard/CamerasCard.vue

@@ -13,11 +13,7 @@ defineProps<{
 
 const driverMode = computed<boolean>({
   get: () => useCameraSettingsStore().isDriverMode,
-  set: (v) =>
-    useCameraSettingsStore().changeCurrentPipelineIndex(
-      v ? -1 : useCameraSettingsStore().currentCameraSettings.lastPipelineIndex || 0,
-      true
-    )
+  set: (v) => useCameraSettingsStore().setDriverMode(v)
 });
 
 const fpsTooLow = computed<boolean>(() => {

photon-client/src/components/dashboard/tabs/InputTab.vue

@@ -130,12 +130,32 @@ const interactiveCols = computed(() =>
       tooltip="Controls blue automatic white balance gain, which affects how the camera captures colors in different conditions"
       @input="(args) => useCameraSettingsStore().changeCurrentPipelineSetting({ cameraBlueGain: args }, false)"
     />
+    <!-- Disable camera orientation as stop gap for Issue 1084 until calibration data gets rotated. https://github.com/PhotonVision/photonvision/issues/1084 -->
+    <v-banner
+      v-show="
+        useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
+        useCameraSettingsStore().currentPipelineSettings.inputImageRotationMode != 0
+      "
+      rounded
+      dark
+      color="red"
+      text-color="white"
+      class="mt-3"
+      icon="mdi-alert-circle-outline"
+    >
+      Warning! A known bug affects rotation of calibrated camera. Turn off rotation here and rotate using
+      cameraToRobotTransform in your robot code.
+    </v-banner>
     <pv-select
       v-model="useCameraSettingsStore().currentPipelineSettings.inputImageRotationMode"
       label="Orientation"
-      tooltip="Rotates the camera stream"
+      tooltip="Rotates the camera stream. Rotation not available when camera has been calibrated."
       :items="cameraRotations"
       :select-cols="interactiveCols"
+      :disabled="
+        useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
+        useCameraSettingsStore().currentPipelineSettings.inputImageRotationMode == 0
+      "
       @input="(args) => useCameraSettingsStore().changeCurrentPipelineSetting({ inputImageRotationMode: args }, false)"
     />
     <pv-select

photon-client/src/stores/settings/CameraSettingsStore.ts

@@ -236,6 +236,13 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
       }
       useStateStore().websocket?.send(payload, true);
     },
+    setDriverMode(isDriverMode: boolean, cameraIndex: number = useStateStore().currentCameraIndex) {
+      const payload = {
+        driverMode: isDriverMode,
+        cameraIndex: cameraIndex
+      };
+      useStateStore().websocket?.send(payload, true);
+    },
     /**
      * Change the currently selected pipeline of the provided camera.
      *

photon-core/src/main/java/org/photonvision/common/logging/LogGroup.java

@@ -23,5 +23,6 @@ public enum LogGroup {
     VisionModule,
     Data,
     General,
-    Config
+    Config,
+    CSCore,
 }

photon-core/src/main/java/org/photonvision/common/logging/Logger.java

@@ -99,6 +99,7 @@ public class Logger {
         levelMap.put(LogGroup.Data, LogLevel.INFO);
         levelMap.put(LogGroup.VisionModule, LogLevel.INFO);
         levelMap.put(LogGroup.Config, LogLevel.INFO);
+        levelMap.put(LogGroup.CSCore, LogLevel.TRACE);
     }
 
     static {
@@ -194,7 +195,7 @@ public class Logger {
         return logLevel.code <= levelMap.get(group).code;
     }
 
-    private void log(String message, LogLevel level) {
+    void log(String message, LogLevel level) {
         if (shouldLog(level)) {
             log(message, level, group, className);
         }

photon-core/src/main/java/org/photonvision/common/logging/PvCSCoreLogger.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.common.logging;
+
+import edu.wpi.first.cscore.CameraServerJNI;
+import java.nio.file.Path;
+
+/** Redirect cscore logs to our logger */
+public class PvCSCoreLogger {
+    private static PvCSCoreLogger INSTANCE;
+
+    public static PvCSCoreLogger getInstance() {
+        if (INSTANCE == null) {
+            INSTANCE = new PvCSCoreLogger();
+        }
+        return INSTANCE;
+    }
+
+    private Logger logger;
+
+    private PvCSCoreLogger() {
+        CameraServerJNI.setLogger(this::logMsg, 7);
+        this.logger = new Logger(getClass(), LogGroup.CSCore);
+    }
+
+    private void logMsg(int level, String file, int line, String msg) {
+        if (level == 20) {
+            logger.info(msg);
+            return;
+        }
+
+        file = Path.of(file).getFileName().toString();
+
+        String levelmsg;
+        LogLevel pvlevel;
+        if (level >= 50) {
+            levelmsg = "CRITICAL";
+            pvlevel = LogLevel.ERROR;
+        } else if (level >= 40) {
+            levelmsg = "ERROR";
+            pvlevel = LogLevel.ERROR;
+        } else if (level >= 30) {
+            levelmsg = "WARNING";
+            pvlevel = LogLevel.WARN;
+        } else if (level >= 20) {
+            levelmsg = "INFO";
+            pvlevel = LogLevel.INFO;
+        } else {
+            levelmsg = "DEBUG";
+            pvlevel = LogLevel.DEBUG;
+        }
+        logger.log(
+                "CS: " + levelmsg + " " + level + ": " + msg + " (" + file + ":" + line + ")", pvlevel);
+    }
+}

photon-core/src/main/java/org/photonvision/jni/RknnDetectorJNI.java

@@ -22,12 +22,12 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.stream.Collectors;
+import org.opencv.core.Mat;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.util.TestUtils;
 import org.photonvision.rknn.RknnJNI;
 import org.photonvision.rknn.RknnJNI.RknnResult;
-import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult;
 
 public class RknnDetectorJNI extends PhotonJNICommon {
@@ -65,16 +65,38 @@ public class RknnDetectorJNI extends PhotonJNICommon {
         long objPointer = -1;
         private List<String> labels;
         private final Object lock = new Object();
-        private static final CopyOnWriteArrayList<Long> detectors = new CopyOnWriteArrayList<>();
+        private static final CopyOnWriteArrayList<RknnObjectDetector> detectors =
+                new CopyOnWriteArrayList<>();
+
+        static volatile boolean hook = false;
 
         public RknnObjectDetector(String modelPath, List<String> labels, RknnJNI.ModelVersion version) {
             synchronized (lock) {
                 objPointer = RknnJNI.create(modelPath, labels.size(), version.ordinal(), -1);
-                detectors.add(objPointer);
-                System.out.println(
-                        "Created " + objPointer + "! Detectors: " + Arrays.toString(detectors.toArray()));
+                detectors.add(this);
+                logger.debug(
+                        "Created detector "
+                                + objPointer
+                                + " from path "
+                                + modelPath
+                                + "! Detectors: "
+                                + Arrays.toString(detectors.toArray()));
             }
             this.labels = labels;
+
+            // the kernel should probably alredy deal with this for us, but I'm gunna be paranoid anyways.
+            if (!hook) {
+                Runtime.getRuntime()
+                        .addShutdownHook(
+                                new Thread(
+                                        () -> {
+                                            System.err.println("Shutdown hook rknn");
+                                            for (var d : detectors) {
+                                                d.release();
+                                            }
+                                        }));
+                hook = true;
+            }
         }
 
         public List<String> getClasses() {
@@ -89,14 +111,14 @@ public class RknnDetectorJNI extends PhotonJNICommon {
          * @param boxThresh Minimum confidence for a box to be added. Basically just confidence
          *     threshold
          */
-        public List<NeuralNetworkPipeResult> detect(CVMat in, double nmsThresh, double boxThresh) {
+        public List<NeuralNetworkPipeResult> detect(Mat in, double nmsThresh, double boxThresh) {
             RknnResult[] ret;
             synchronized (lock) {
                 // We can technically be asked to detect and the lock might be acquired _after_ release has
                 // been called. This would mean objPointer would be invalid which would call everything to
                 // explode.
                 if (objPointer > 0) {
-                    ret = RknnJNI.detect(objPointer, in.getMat().getNativeObjAddr(), nmsThresh, boxThresh);
+                    ret = RknnJNI.detect(objPointer, in.getNativeObjAddr(), nmsThresh, boxThresh);
                 } else {
                     logger.warn("Detect called after destroy -- giving up");
                     return List.of();
@@ -114,7 +136,7 @@ public class RknnDetectorJNI extends PhotonJNICommon {
             synchronized (lock) {
                 if (objPointer > 0) {
                     RknnJNI.destroy(objPointer);
-                    detectors.remove(objPointer);
+                    detectors.remove(this);
                     System.out.println(
                             "Killed " + objPointer + "! Detectors: " + Arrays.toString(detectors.toArray()));
                     objPointer = -1;
@@ -124,14 +146,4 @@ public class RknnDetectorJNI extends PhotonJNICommon {
             }
         }
     }
-
-    // public static void createRknnDetector() {
-    //     objPointer =
-    //             RknnJNI.create(
-    //                     NeuralNetworkModelManager.getInstance()
-    //                             .getDefaultRknnModel()
-    //                             .getAbsolutePath()
-    //                             .toString(),
-    //                     NeuralNetworkModelManager.getInstance().getLabels().size());
-    // }
 }

photon-core/src/main/java/org/photonvision/vision/aruco/PhotonArucoDetector.java

@@ -23,16 +23,37 @@ import java.util.Comparator;
 import org.opencv.core.Mat;
 import org.opencv.objdetect.ArucoDetector;
 import org.opencv.objdetect.DetectorParameters;
+import org.opencv.objdetect.Dictionary;
 import org.opencv.objdetect.Objdetect;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
+import org.photonvision.vision.opencv.Releasable;
 
 /** This class wraps an {@link ArucoDetector} for convenience. */
-public class PhotonArucoDetector {
+public class PhotonArucoDetector implements Releasable {
     private static final Logger logger = new Logger(PhotonArucoDetector.class, LogGroup.VisionModule);
 
-    private final ArucoDetector detector =
-            new ArucoDetector(Objdetect.getPredefinedDictionary(Objdetect.DICT_APRILTAG_16h5));
+    private static class ArucoDetectorHack extends ArucoDetector {
+        public ArucoDetectorHack(Dictionary predefinedDictionary) {
+            super(predefinedDictionary);
+        }
+
+        // avoid double-free by keeping track of this ourselves (ew)
+        private boolean freed = false;
+
+        @Override
+        public void finalize() throws Throwable {
+            if (freed) {
+                return;
+            }
+
+            super.finalize();
+            freed = true;
+        }
+    }
+
+    private final ArucoDetectorHack detector =
+            new ArucoDetectorHack(Objdetect.getPredefinedDictionary(Objdetect.DICT_APRILTAG_16h5));
 
     private final Mat ids = new Mat();
     private final ArrayList<Mat> cornerMats = new ArrayList<>();
@@ -95,4 +116,16 @@ public class PhotonArucoDetector {
 
         return results;
     }
+
+    @Override
+    public void release() {
+        try {
+            detector.finalize();
+        } catch (Throwable e) {
+            logger.error("Exception destroying PhotonArucoDetector", e);
+        }
+        ids.release();
+        for (var m : cornerMats) m.release();
+        cornerMats.clear();
+    }
 }

photon-core/src/main/java/org/photonvision/vision/camera/CameraInfo.java

@@ -108,7 +108,7 @@ public class CameraInfo extends UsbCameraInfo {
     public String toString() {
         return "CameraInfo [cameraType="
                 + cameraType
-                + "baseName="
+                + ", baseName="
                 + getBaseName()
                 + ", vid="
                 + vendorId

photon-core/src/main/java/org/photonvision/vision/camera/USBCameraSource.java

@@ -270,9 +270,13 @@ public class USBCameraSource extends VisionSource {
                         if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.ArduOV9281)) {
                             propMin = 1;
                             propMax = 75;
+                        } else if (getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.ArduOV2311)) {
+                            propMin = 1;
+                            propMax = 140;
                         }
 
                         var exposure_manual_val = MathUtils.map(Math.round(exposure), 0, 100, propMin, propMax);
+                        logger.debug("Setting camera exposure to " + exposure_manual_val);
                         prop.set((int) exposure_manual_val);
                     } else {
                         scaledExposure = (int) Math.round(exposure);

photon-core/src/main/java/org/photonvision/vision/frame/provider/FileFrameProvider.java

@@ -33,7 +33,7 @@ import org.photonvision.vision.opencv.CVMat;
  * path}.
  */
 public class FileFrameProvider extends CpuImageProcessor {
-    public static final int MAX_FPS = 5;
+    public static final int MAX_FPS = 10;
     private static int count = 0;
 
     private final int thisIndex = count++;

photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java

@@ -21,11 +21,13 @@ import edu.wpi.first.apriltag.AprilTagDetection;
 import edu.wpi.first.apriltag.AprilTagDetector;
 import java.util.List;
 import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.opencv.Releasable;
 import org.photonvision.vision.pipe.CVPipe;
 
 public class AprilTagDetectionPipe
-        extends CVPipe<CVMat, List<AprilTagDetection>, AprilTagDetectionPipeParams> {
-    private final AprilTagDetector m_detector = new AprilTagDetector();
+        extends CVPipe<CVMat, List<AprilTagDetection>, AprilTagDetectionPipeParams>
+        implements Releasable {
+    private AprilTagDetector m_detector = new AprilTagDetector();
 
     public AprilTagDetectionPipe() {
         super();
@@ -40,6 +42,10 @@ public class AprilTagDetectionPipe
             return List.of();
         }
 
+        if (m_detector == null) {
+            throw new RuntimeException("Apriltag detector was released!");
+        }
+
         var ret = m_detector.detect(in.getMat());
 
         if (ret == null) {
@@ -60,4 +66,10 @@ public class AprilTagDetectionPipe
 
         super.setParams(newParams);
     }
+
+    @Override
+    public void release() {
+        m_detector.close();
+        m_detector = null;
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagPoseEstimatorPipe.java

@@ -25,13 +25,15 @@ import org.opencv.calib3d.Calib3d;
 import org.opencv.core.MatOfPoint2f;
 import org.opencv.core.Point;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
+import org.photonvision.vision.opencv.Releasable;
 import org.photonvision.vision.pipe.CVPipe;
 
 public class AprilTagPoseEstimatorPipe
         extends CVPipe<
                 AprilTagDetection,
                 AprilTagPoseEstimate,
-                AprilTagPoseEstimatorPipe.AprilTagPoseEstimatorPipeParams> {
+                AprilTagPoseEstimatorPipe.AprilTagPoseEstimatorPipeParams>
+        implements Releasable {
     private final AprilTagPoseEstimator m_poseEstimator =
             new AprilTagPoseEstimator(new AprilTagPoseEstimator.Config(0, 0, 0, 0, 0));
 
@@ -92,6 +94,11 @@ public class AprilTagPoseEstimatorPipe
         super.setParams(newParams);
     }
 
+    @Override
+    public void release() {
+        temp.release();
+    }
+
     public static class AprilTagPoseEstimatorPipeParams {
         final AprilTagPoseEstimator.Config config;
         final CameraCalibrationCoefficients calibration;

photon-core/src/main/java/org/photonvision/vision/pipe/impl/ArucoDetectionPipe.java

@@ -29,10 +29,12 @@ import org.opencv.objdetect.Objdetect;
 import org.photonvision.vision.aruco.ArucoDetectionResult;
 import org.photonvision.vision.aruco.PhotonArucoDetector;
 import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.opencv.Releasable;
 import org.photonvision.vision.pipe.CVPipe;
 
 public class ArucoDetectionPipe
-        extends CVPipe<CVMat, List<ArucoDetectionResult>, ArucoDetectionPipeParams> {
+        extends CVPipe<CVMat, List<ArucoDetectionResult>, ArucoDetectionPipeParams>
+        implements Releasable {
     // ArucoDetector wrapper class
     private final PhotonArucoDetector photonDetector = new PhotonArucoDetector();
 
@@ -131,4 +133,9 @@ public class ArucoDetectionPipe
         var pt2 = new Point(corner.x + windowSize, corner.y + windowSize);
         Imgproc.rectangle(outputMat, pt1, pt2, new Scalar(0, 0, 255), thickness);
     }
+
+    @Override
+    public void release() {
+        photonDetector.release();
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipe/impl/ArucoPoseEstimatorPipe.java

@@ -32,13 +32,15 @@ import org.opencv.core.MatOfPoint3f;
 import org.opencv.core.Point3;
 import org.photonvision.vision.aruco.ArucoDetectionResult;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
+import org.photonvision.vision.opencv.Releasable;
 import org.photonvision.vision.pipe.CVPipe;
 
 public class ArucoPoseEstimatorPipe
         extends CVPipe<
                 ArucoDetectionResult,
                 AprilTagPoseEstimate,
-                ArucoPoseEstimatorPipe.ArucoPoseEstimatorPipeParams> {
+                ArucoPoseEstimatorPipe.ArucoPoseEstimatorPipeParams>
+        implements Releasable {
     // image points of marker corners
     private final MatOfPoint2f imagePoints = new MatOfPoint2f(Mat.zeros(4, 1, CvType.CV_32FC2));
     // rvec/tvec estimations from solvepnp
@@ -117,6 +119,18 @@ public class ArucoPoseEstimatorPipe
         super.setParams(newParams);
     }
 
+    @Override
+    public void release() {
+        imagePoints.release();
+        for (var m : rvecs) m.release();
+        rvecs.clear();
+        for (var m : tvecs) m.release();
+        tvecs.clear();
+        rvec.release();
+        tvec.release();
+        reprojectionErrors.release();
+    }
+
     public static class ArucoPoseEstimatorPipeParams {
         final CameraCalibrationCoefficients calibration;
         final double tagSize;

photon-core/src/main/java/org/photonvision/vision/pipe/impl/Calibrate3dPipe.java

@@ -38,13 +38,26 @@ import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
 import org.photonvision.vision.calibration.CameraLensModel;
 import org.photonvision.vision.calibration.JsonImageMat;
 import org.photonvision.vision.calibration.JsonMatOfDouble;
+import org.photonvision.vision.frame.FrameStaticProperties;
 import org.photonvision.vision.pipe.CVPipe;
+import org.photonvision.vision.pipe.impl.FindBoardCornersPipe.FindBoardCornersPipeResult;
 
 public class Calibrate3dPipe
         extends CVPipe<
-                List<FindBoardCornersPipe.FindBoardCornersPipeResult>,
+                Calibrate3dPipe.CalibrationInput,
                 CameraCalibrationCoefficients,
                 Calibrate3dPipe.CalibratePipeParams> {
+    public static class CalibrationInput {
+        final List<FindBoardCornersPipe.FindBoardCornersPipeResult> observations;
+        final FrameStaticProperties imageProps;
+
+        public CalibrationInput(
+                List<FindBoardCornersPipeResult> observations, FrameStaticProperties imageProps) {
+            this.observations = observations;
+            this.imageProps = imageProps;
+        }
+    }
+
     // For logging
     private static final Logger logger = new Logger(Calibrate3dPipe.class, LogGroup.General);
 
@@ -63,10 +76,9 @@ public class Calibrate3dPipe
      * @return Result of processing.
      */
     @Override
-    protected CameraCalibrationCoefficients process(
-            List<FindBoardCornersPipe.FindBoardCornersPipeResult> in) {
-        in =
-                in.stream()
+    protected CameraCalibrationCoefficients process(CalibrationInput in) {
+        var filteredIn =
+                in.observations.stream()
                         .filter(
                                 it ->
                                         it != null
@@ -79,17 +91,21 @@ public class Calibrate3dPipe
         var start = System.nanoTime();
         if (MrCalJNILoader.getInstance().isLoaded() && params.useMrCal) {
             logger.debug("Calibrating with mrcal!");
-            ret = calibrateMrcal(in);
+            ret =
+                    calibrateMrcal(
+                            filteredIn, in.imageProps.horizontalFocalLength, in.imageProps.verticalFocalLength);
         } else {
             logger.debug("Calibrating with opencv!");
-            ret = calibrateOpenCV(in);
+            ret =
+                    calibrateOpenCV(
+                            filteredIn, in.imageProps.horizontalFocalLength, in.imageProps.verticalFocalLength);
         }
         var dt = System.nanoTime() - start;
 
         if (ret != null)
             logger.info(
                     "CALIBRATION SUCCESS for res "
-                            + in.get(0).size
+                            + in.observations.get(0).size
                             + " in "
                             + dt / 1e6
                             + "ms! camMatrix: \n"
@@ -103,7 +119,7 @@ public class Calibrate3dPipe
     }
 
     protected CameraCalibrationCoefficients calibrateOpenCV(
-            List<FindBoardCornersPipe.FindBoardCornersPipeResult> in) {
+            List<FindBoardCornersPipe.FindBoardCornersPipeResult> in, double fxGuess, double fyGuess) {
         List<Mat> objPoints = in.stream().map(it -> it.objectPoints).collect(Collectors.toList());
         List<Mat> imgPts = in.stream().map(it -> it.imagePoints).collect(Collectors.toList());
         if (objPoints.size() != imgPts.size()) {
@@ -111,30 +127,32 @@ public class Calibrate3dPipe
             return null;
         }
 
-        Mat cameraMatrix = new Mat();
+        Mat cameraMatrix = new Mat(3, 3, CvType.CV_64F);
         MatOfDouble distortionCoefficients = new MatOfDouble();
         List<Mat> rvecs = new ArrayList<>();
         List<Mat> tvecs = new ArrayList<>();
 
-        // RMS of the calibration
-        double calibrationAccuracy;
+        // initial camera matrix guess
+        double cx = (in.get(0).size.width / 2.0) - 0.5;
+        double cy = (in.get(0).size.width / 2.0) - 0.5;
+        cameraMatrix.put(0, 0, new double[] {fxGuess, 0, cx, 0, fyGuess, cy, 0, 0, 1});
 
         try {
             // FindBoardCorners pipe outputs all the image points, object points, and frames to calculate
             // imageSize from, other parameters are output Mats
 
-            calibrationAccuracy =
-                    Calib3d.calibrateCameraExtended(
-                            objPoints,
-                            imgPts,
-                            new Size(in.get(0).size.width, in.get(0).size.height),
-                            cameraMatrix,
-                            distortionCoefficients,
-                            rvecs,
-                            tvecs,
-                            stdDeviationsIntrinsics,
-                            stdDeviationsExtrinsics,
-                            perViewErrors);
+            Calib3d.calibrateCameraExtended(
+                    objPoints,
+                    imgPts,
+                    new Size(in.get(0).size.width, in.get(0).size.height),
+                    cameraMatrix,
+                    distortionCoefficients,
+                    rvecs,
+                    tvecs,
+                    stdDeviationsIntrinsics,
+                    stdDeviationsExtrinsics,
+                    perViewErrors,
+                    Calib3d.CALIB_USE_LU + Calib3d.CALIB_USE_INTRINSIC_GUESS);
         } catch (Exception e) {
             logger.error("Calibration failed!", e);
             e.printStackTrace();
@@ -164,23 +182,22 @@ public class Calibrate3dPipe
     }
 
     protected CameraCalibrationCoefficients calibrateMrcal(
-            List<FindBoardCornersPipe.FindBoardCornersPipeResult> in) {
+            List<FindBoardCornersPipe.FindBoardCornersPipeResult> in, double fxGuess, double fyGuess) {
         List<MatOfPoint2f> corner_locations =
                 in.stream().map(it -> it.imagePoints).map(MatOfPoint2f::new).collect(Collectors.toList());
 
         int imageWidth = (int) in.get(0).size.width;
         int imageHeight = (int) in.get(0).size.height;
-        final double FOCAL_LENGTH_GUESS = 1200;
 
-        MrCalResult result =
-                MrCalJNI.calibrateCamera(
-                        corner_locations,
-                        params.boardWidth,
-                        params.boardHeight,
-                        params.squareSize,
-                        imageWidth,
-                        imageHeight,
-                        FOCAL_LENGTH_GUESS);
+        MrCalResult result = null;
+                // MrCalJNI.calibrateCamera(
+                //         corner_locations,
+                //         params.boardWidth,
+                //         params.boardHeight,
+                //         params.squareSize,
+                //         imageWidth,
+                //         imageHeight,
+                //         (fxGuess + fyGuess) / 2.0);
 
         // intrinsics are fx fy cx cy from mrcal
         JsonMatOfDouble cameraMatrixMat =

photon-core/src/main/java/org/photonvision/vision/pipe/impl/Calibrate3dPipeline.java

@@ -36,6 +36,7 @@ import org.photonvision.vision.frame.FrameThresholdType;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.opencv.ImageRotationMode;
 import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
+import org.photonvision.vision.pipe.impl.Calibrate3dPipe.CalibrationInput;
 import org.photonvision.vision.pipe.impl.FindBoardCornersPipe.FindBoardCornersPipeResult;
 import org.photonvision.vision.pipeline.CVPipeline;
 import org.photonvision.vision.pipeline.Calibration3dPipelineSettings;
@@ -176,7 +177,8 @@ public class Calibrate3dPipeline
 
         /*Pass the board corners to the pipe, which will check again to see if all boards are valid
         and returns the corresponding image and object points*/
-        calibrationOutput = calibrate3dPipe.run(foundCornersList);
+        calibrationOutput =
+                calibrate3dPipe.run(new CalibrationInput(foundCornersList, frameStaticProperties));
 
         this.calibrating = false;
 
@@ -229,4 +231,9 @@ public class Calibrate3dPipeline
     public CameraCalibrationCoefficients cameraCalibrationCoefficients() {
         return calibrationOutput.output;
     }
+
+    @Override
+    public void release() {
+        // we never actually need to give resources up since pipelinemanager only makes one of us
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterObjectDetectionsPipe.java

@@ -41,7 +41,7 @@ public class FilterObjectDetectionsPipe
     }
 
     private void filterContour(NeuralNetworkPipeResult contour) {
-        var boc = contour.box;
+        var boc = contour.bbox;
 
         // Area filtering
         double areaPercentage = boc.area() / params.getFrameStaticProperties().imageArea * 100.0;

photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java

@@ -20,13 +20,13 @@ package org.photonvision.vision.pipe.impl;
 import org.opencv.core.Rect2d;
 
 public class NeuralNetworkPipeResult {
-    public NeuralNetworkPipeResult(Rect2d box2, Integer classIdx, Float confidence) {
-        box = box2;
+    public NeuralNetworkPipeResult(Rect2d boundingBox, int classIdx, double confidence) {
+        bbox = boundingBox;
         this.classIdx = classIdx;
         this.confidence = confidence;
     }
 
     public final int classIdx;
-    public final Rect2d box;
+    public final Rect2d bbox;
     public final double confidence;
 }

photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java

@@ -17,8 +17,17 @@
 
 package org.photonvision.vision.pipe.impl;
 
+import java.awt.Color;
+import java.util.ArrayList;
 import java.util.List;
+import org.opencv.core.Core;
+import org.opencv.core.Mat;
+import org.opencv.core.Rect2d;
+import org.opencv.core.Scalar;
+import org.opencv.core.Size;
+import org.opencv.imgproc.Imgproc;
 import org.photonvision.common.configuration.NeuralNetworkModelManager;
+import org.photonvision.common.util.ColorHelper;
 import org.photonvision.jni.RknnDetectorJNI.RknnObjectDetector;
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.opencv.Releasable;
@@ -30,8 +39,10 @@ public class RknnDetectionPipe
     private RknnObjectDetector detector;
 
     public RknnDetectionPipe() {
-        // For now this is hard-coded to defaults. Should be refactored into set pipe params, though.
-        // And ideally a little wrapper helper for only changing native stuff on content change created.
+        // For now this is hard-coded to defaults. Should be refactored into set pipe
+        // params, though.
+        // And ideally a little wrapper helper for only changing native stuff on content
+        // change created.
         this.detector =
                 new RknnObjectDetector(
                         NeuralNetworkModelManager.getInstance().getDefaultRknnModel().getAbsolutePath(),
@@ -39,6 +50,18 @@ public class RknnDetectionPipe
                         NeuralNetworkModelManager.getInstance().getModelVersion());
     }
 
+    private static class Letterbox {
+        double dx;
+        double dy;
+        double scale;
+
+        public Letterbox(double dx, double dy, double scale) {
+            this.dx = dx;
+            this.dy = dy;
+            this.scale = scale;
+        }
+    }
+
     @Override
     protected List<NeuralNetworkPipeResult> process(CVMat in) {
         var frame = in.getMat();
@@ -48,7 +71,67 @@ public class RknnDetectionPipe
             return List.of();
         }
 
-        return detector.detect(in, params.nms, params.confidence);
+        // letterbox
+        var letterboxed = new Mat();
+        var scale =
+                letterbox(frame, letterboxed, new Size(640, 640), ColorHelper.colorToScalar(Color.GRAY));
+
+        if (letterboxed.width() != 640 || letterboxed.height() != 640) {
+            // huh whack give up lol
+            throw new RuntimeException("RGA bugged but still wrong size");
+        }
+        var ret = detector.detect(letterboxed, params.nms, params.confidence);
+        letterboxed.release();
+
+        return resizeDetections(ret, scale);
+    }
+
+    private List<NeuralNetworkPipeResult> resizeDetections(
+            List<NeuralNetworkPipeResult> unscaled, Letterbox letterbox) {
+        var ret = new ArrayList<NeuralNetworkPipeResult>();
+
+        for (var t : unscaled) {
+            var scale = 1.0 / letterbox.scale;
+            var boundingBox = t.bbox;
+            double x = (boundingBox.x - letterbox.dx) * scale;
+            double y = (boundingBox.y - letterbox.dy) * scale;
+            double width = boundingBox.width * scale;
+            double height = boundingBox.height * scale;
+
+            ret.add(
+                    new NeuralNetworkPipeResult(new Rect2d(x, y, width, height), t.classIdx, t.confidence));
+        }
+
+        return ret;
+    }
+
+    private static Letterbox letterbox(Mat frame, Mat letterboxed, Size newShape, Scalar color) {
+        // from https://github.com/ultralytics/yolov5/issues/8427#issuecomment-1172469631
+        var frameSize = frame.size();
+        var r = Math.min(newShape.height / frameSize.height, newShape.width / frameSize.width);
+
+        var newUnpad = new Size(Math.round(frameSize.width * r), Math.round(frameSize.height * r));
+
+        if (!(frameSize.equals(newUnpad))) {
+            Imgproc.resize(frame, letterboxed, newUnpad, Imgproc.INTER_LINEAR);
+        } else {
+            frame.copyTo(letterboxed);
+        }
+
+        var dw = newShape.width - newUnpad.width;
+        var dh = newShape.height - newUnpad.height;
+
+        dw /= 2;
+        dh /= 2;
+
+        int top = (int) (Math.round(dh - 0.1f));
+        int bottom = (int) (Math.round(dh + 0.1f));
+        int left = (int) (Math.round(dw - 0.1f));
+        int right = (int) (Math.round(dw + 0.1f));
+        Core.copyMakeBorder(
+                letterboxed, letterboxed, top, bottom, left, right, Core.BORDER_CONSTANT, color);
+
+        return new Letterbox(dw, dh, r);
     }
 
     public static class RknnDetectionPipeParams {

photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java

@@ -221,4 +221,11 @@ public class AprilTagPipeline extends CVPipeline<CVPipelineResult, AprilTagPipel
 
         return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, multiTagResult, frame);
     }
+
+    @Override
+    public void release() {
+        aprilTagDetectionPipe.release();
+        singleTagPoseEstimatorPipe.release();
+        super.release();
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipeline/ArucoPipeline.java

@@ -60,8 +60,8 @@ import org.photonvision.vision.target.TrackedTarget;
 import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;
 
 public class ArucoPipeline extends CVPipeline<CVPipelineResult, ArucoPipelineSettings> {
-    private final ArucoDetectionPipe arucoDetectionPipe = new ArucoDetectionPipe();
-    private final ArucoPoseEstimatorPipe singleTagPoseEstimatorPipe = new ArucoPoseEstimatorPipe();
+    private ArucoDetectionPipe arucoDetectionPipe = new ArucoDetectionPipe();
+    private ArucoPoseEstimatorPipe singleTagPoseEstimatorPipe = new ArucoPoseEstimatorPipe();
     private final MultiTargetPNPPipe multiTagPNPPipe = new MultiTargetPNPPipe();
     private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
 
@@ -250,4 +250,13 @@ public class ArucoPipeline extends CVPipeline<CVPipelineResult, ArucoPipelineSet
                 windowSize,
                 constant);
     }
+
+    @Override
+    public void release() {
+        arucoDetectionPipe.release();
+        singleTagPoseEstimatorPipe.release();
+        arucoDetectionPipe = null;
+        singleTagPoseEstimatorPipe = null;
+        super.release();
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java

@@ -34,6 +34,9 @@ public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelin
 
     private final FrameThresholdType thresholdType;
 
+    // So releaseable doesn't keep track of if we double-free something. so (ew) remember that here
+    protected volatile boolean released = false;
+
     public CVPipeline(FrameThresholdType thresholdType) {
         this.thresholdType = thresholdType;
     }
@@ -64,6 +67,9 @@ public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelin
     }
 
     public R run(Frame frame, QuirkyCamera cameraQuirks) {
+        if (released) {
+            throw new RuntimeException("Pipeline use-after-free!");
+        }
         if (settings == null) {
             throw new RuntimeException("No settings provided for pipeline!");
         }
@@ -85,5 +91,7 @@ public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelin
      * switch. Stubbed out, but override if needed.
      */
     @Override
-    public void release() {}
+    public void release() {
+        released = true;
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java

@@ -88,4 +88,9 @@ public class DriverModePipeline
                 fps,
                 new Frame(frame.processedImage, frame.colorImage, frame.type, frame.frameStaticProperties));
     }
+
+    @Override
+    public void release() {
+        // we never actually need to give resources up since pipelinemanager only makes one of us
+    }
 }

photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java

@@ -131,5 +131,6 @@ public class ObjectDetectionPipeline
     @Override
     public void release() {
         rknnPipe.release();
+        super.release();
     }
 }

photon-core/src/main/java/org/photonvision/vision/processes/CameraMatchingOptions.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.processes;
+
+import java.util.List;
+import org.photonvision.vision.camera.CameraType;
+
+public class CameraMatchingOptions {
+    public CameraMatchingOptions(
+            boolean checkUSBPath,
+            boolean checkVidPid,
+            boolean checkBaseName,
+            boolean checkPath,
+            CameraType... allowedTypes) {
+        this.checkUSBPath = checkUSBPath;
+        this.checkVidPid = checkVidPid;
+        this.checkBaseName = checkBaseName;
+        this.checkPath = checkPath;
+        this.allowedTypes = List.of(allowedTypes);
+    }
+
+    public final boolean checkUSBPath;
+    public final boolean checkVidPid;
+    public final boolean checkBaseName;
+    public final boolean checkPath;
+    public final List<CameraType> allowedTypes;
+
+    @Override
+    public String toString() {
+        return "CameraMatchingOptions [checkUSBPath="
+                + checkUSBPath
+                + ", checkVidPid="
+                + checkVidPid
+                + ", checkBaseName="
+                + checkBaseName
+                + ", checkPath="
+                + checkPath
+                + ", allowedTypes="
+                + allowedTypes
+                + "]";
+    }
+}

photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java

@@ -61,7 +61,8 @@ public class PipelineManager {
     PipelineManager(
             DriverModePipelineSettings driverSettings,
             List<CVPipelineSettings> userPipelines,
-            String uniqueName) {
+            String uniqueName,
+            int defaultIndex) {
         this.userPipelineSettings = new ArrayList<>(userPipelines);
         // This is to respect the default res idx for vendor cameras
 
@@ -70,10 +71,19 @@ public class PipelineManager {
         if (userPipelines.isEmpty()) addPipeline(PipelineType.Reflective);
 
         calibration3dPipeline = new Calibrate3dPipeline(uniqueName);
+
+        // We know that at this stage, VisionRunner hasn't yet started so we're good to do this from
+        // this thread
+        this.setIndex(defaultIndex);
+        updatePipelineFromRequested();
     }
 
     public PipelineManager(CameraConfiguration config) {
-        this(config.driveModeSettings, config.pipelineSettings, config.uniqueName);
+        this(
+                config.driveModeSettings,
+                config.pipelineSettings,
+                config.uniqueName,
+                config.currentPipelineIndex);
     }
 
     /**
@@ -148,6 +158,7 @@ public class PipelineManager {
      * @return The currently active pipeline.
      */
     public CVPipeline getCurrentPipeline() {
+        updatePipelineFromRequested();
         if (currentPipelineIndex < 0) {
             switch (currentPipelineIndex) {
                 case CAL_3D_INDEX:
@@ -170,6 +181,16 @@ public class PipelineManager {
         return getPipelineSettings(currentPipelineIndex);
     }
 
+    private volatile int requestedIndex = 0;
+
+    /**
+     * Grab the currently requested pipeline index. The VisionRunner may not have changed over to this
+     * pipeline yet.
+     */
+    public int getRequestedIndex() {
+        return requestedIndex;
+    }
+
     /**
      * Internal method for setting the active pipeline. <br>
      * <br>
@@ -179,6 +200,22 @@ public class PipelineManager {
      * @param newIndex Index of pipeline to be active
      */
     private void setPipelineInternal(int newIndex) {
+        requestedIndex = newIndex;
+    }
+
+    /**
+     * Based on a requested pipeline index, create/destroy pipelines as necessary. We do this as a
+     * side effect of the main thread that calls getCurrentPipeline to avoid race conditions between
+     * server threads and the VisionRunner TODO: this should be refactored. Shame Java doesn't have
+     * RAII
+     */
+    private void updatePipelineFromRequested() {
+        int newIndex = requestedIndex;
+        if (newIndex == currentPipelineIndex) {
+            // nothing to do, probably no change -- give up
+            return;
+        }
+
         if (newIndex < 0 && currentPipelineIndex >= 0) {
             // Transitioning to a built-in pipe, save off the current user one
             lastUserPipelineIdx = currentPipelineIndex;
@@ -189,44 +226,10 @@ public class PipelineManager {
             return;
         }
 
-        // Cleanup potential old native resources before swapping over
-        if (currentUserPipeline != null) {
-            currentUserPipeline.release();
-        }
-
         currentPipelineIndex = newIndex;
-        if (newIndex >= 0) {
-            var desiredPipelineSettings = userPipelineSettings.get(currentPipelineIndex);
-            switch (desiredPipelineSettings.pipelineType) {
-                case Reflective:
-                    logger.debug("Creating Reflective pipeline");
-                    currentUserPipeline =
-                            new ReflectivePipeline((ReflectivePipelineSettings) desiredPipelineSettings);
-                    break;
-                case ColoredShape:
-                    logger.debug("Creating ColoredShape pipeline");
-                    currentUserPipeline =
-                            new ColoredShapePipeline((ColoredShapePipelineSettings) desiredPipelineSettings);
-                    break;
-                case AprilTag:
-                    logger.debug("Creating AprilTag pipeline");
-                    currentUserPipeline =
-                            new AprilTagPipeline((AprilTagPipelineSettings) desiredPipelineSettings);
-                    break;
 
-                case Aruco:
-                    logger.debug("Creating Aruco Pipeline");
-                    currentUserPipeline = new ArucoPipeline((ArucoPipelineSettings) desiredPipelineSettings);
-                    break;
-                case ObjectDetection:
-                    logger.debug("Creating ObjectDetection Pipeline");
-                    currentUserPipeline =
-                            new ObjectDetectionPipeline(
-                                    (ObjectDetectionPipelineSettings) desiredPipelineSettings);
-                default:
-                    // Can be calib3d or drivermode, both of which are special cases
-                    break;
-            }
+        if (newIndex >= 0) {
+            recreateUserPipeline();
         }
 
         DataChangeService.getInstance()
@@ -235,6 +238,48 @@ public class PipelineManager {
                                 "fullsettings", ConfigManager.getInstance().getConfig().toHashMap()));
     }
 
+    /**
+     * Recreate the current user pipeline with the current pipeline index. Useful to force a
+     * recreation after changing pipeline type
+     */
+    private void recreateUserPipeline() {
+        // Cleanup potential old native resources before swapping over from a user pipeline
+        if (currentUserPipeline != null && !(currentPipelineIndex < 0)) {
+            currentUserPipeline.release();
+        }
+
+        var desiredPipelineSettings = userPipelineSettings.get(currentPipelineIndex);
+        switch (desiredPipelineSettings.pipelineType) {
+            case Reflective:
+                logger.debug("Creating Reflective pipeline");
+                currentUserPipeline =
+                        new ReflectivePipeline((ReflectivePipelineSettings) desiredPipelineSettings);
+                break;
+            case ColoredShape:
+                logger.debug("Creating ColoredShape pipeline");
+                currentUserPipeline =
+                        new ColoredShapePipeline((ColoredShapePipelineSettings) desiredPipelineSettings);
+                break;
+            case AprilTag:
+                logger.debug("Creating AprilTag pipeline");
+                currentUserPipeline =
+                        new AprilTagPipeline((AprilTagPipelineSettings) desiredPipelineSettings);
+                break;
+
+            case Aruco:
+                logger.debug("Creating Aruco Pipeline");
+                currentUserPipeline = new ArucoPipeline((ArucoPipelineSettings) desiredPipelineSettings);
+                break;
+            case ObjectDetection:
+                logger.debug("Creating ObjectDetection Pipeline");
+                currentUserPipeline =
+                        new ObjectDetectionPipeline((ObjectDetectionPipelineSettings) desiredPipelineSettings);
+            default:
+                // Can be calib3d or drivermode, both of which are special cases
+                break;
+        }
+    }
+
     /**
      * Enters or exits calibration mode based on the parameter. <br>
      * <br>
@@ -465,5 +510,6 @@ public class PipelineManager {
         userPipelineSettings.set(idx, newSettings);
         setPipelineInternal(idx);
         reassignIndexes();
+        recreateUserPipeline();
     }
 }

photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java

@@ -143,7 +143,7 @@ public class VisionModule {
         ntConsumer =
                 new NTDataPublisher(
                         visionSource.getSettables().getConfiguration().nickname,
-                        pipelineManager::getCurrentPipelineIndex,
+                        pipelineManager::getRequestedIndex,
                         this::setPipeline,
                         pipelineManager::getDriverMode,
                         this::setDriverMode);
@@ -156,6 +156,7 @@ public class VisionModule {
                 (result) ->
                         lastPipelineResultBestTarget = result.hasTargets() ? result.targets.get(0) : null);
 
+        // Sync VisionModule state with the first pipeline index
         setPipeline(visionSource.getSettables().getConfiguration().currentPipelineIndex);
 
         // Set vendor FOV
@@ -321,7 +322,7 @@ public class VisionModule {
 
     void changePipelineType(int newType) {
         pipelineManager.changePipelineType(newType);
-        setPipeline(pipelineManager.getCurrentPipelineIndex());
+        setPipeline(pipelineManager.getRequestedIndex());
         saveAndBroadcastAll();
     }
 
@@ -329,9 +330,7 @@ public class VisionModule {
         pipelineManager.setDriverMode(isDriverMode);
         setVisionLEDs(!isDriverMode);
         setPipeline(
-                isDriverMode
-                        ? PipelineManager.DRIVERMODE_INDEX
-                        : pipelineManager.getCurrentPipelineIndex());
+                isDriverMode ? PipelineManager.DRIVERMODE_INDEX : pipelineManager.getRequestedIndex());
         saveAndBroadcastAll();
     }
 
@@ -385,7 +384,7 @@ public class VisionModule {
         var ret = pipelineManager.calibration3dPipeline.tryCalibration();
         pipelineManager.setCalibrationMode(false);
 
-        setPipeline(pipelineManager.getCurrentPipelineIndex());
+        setPipeline(pipelineManager.getRequestedIndex());
 
         if (ret != null) {
             logger.debug("Saving calibration...");
@@ -447,7 +446,7 @@ public class VisionModule {
         setVisionLEDs(pipelineSettings.ledMode);
 
         visionSource.getSettables().getConfiguration().currentPipelineIndex =
-                pipelineManager.getCurrentPipelineIndex();
+                pipelineManager.getRequestedIndex();
 
         return true;
     }
@@ -511,7 +510,7 @@ public class VisionModule {
         ret.uniqueName = visionSource.getSettables().getConfiguration().uniqueName;
         ret.currentPipelineSettings =
                 SerializationUtils.objectToHashMap(pipelineManager.getCurrentPipelineSettings());
-        ret.currentPipelineIndex = pipelineManager.getCurrentPipelineIndex();
+        ret.currentPipelineIndex = pipelineManager.getRequestedIndex();
         ret.pipelineNicknames = pipelineManager.getPipelineNicknames();
         ret.cameraQuirks = visionSource.getSettables().getConfiguration().cameraQuirks;
 
@@ -553,7 +552,7 @@ public class VisionModule {
         var config = visionSource.getSettables().getConfiguration();
         config.setPipelineSettings(pipelineManager.userPipelineSettings);
         config.driveModeSettings = pipelineManager.driverModePipeline.getSettings();
-        config.currentPipelineIndex = Math.max(pipelineManager.getCurrentPipelineIndex(), -1);
+        config.currentPipelineIndex = Math.max(pipelineManager.getRequestedIndex(), -1);
 
         return config;
     }

photon-core/src/main/java/org/photonvision/vision/processes/VisionModuleChangeSubscriber.java

@@ -88,7 +88,7 @@ public class VisionModuleChangeSubscriber extends DataChangeSubscriber {
                         parentModule.saveAndBroadcastAll();
                         return;
                     case "deleteCurrPipeline":
-                        var indexToDelete = parentModule.pipelineManager.getCurrentPipelineIndex();
+                        var indexToDelete = parentModule.pipelineManager.getRequestedIndex();
                         logger.info("Deleting current pipe at index " + indexToDelete);
                         int newIndex = parentModule.pipelineManager.removePipeline(indexToDelete);
                         parentModule.setPipeline(newIndex);
@@ -96,7 +96,7 @@ public class VisionModuleChangeSubscriber extends DataChangeSubscriber {
                         return;
                     case "changePipeline": // change active pipeline
                         var index = (Integer) newPropValue;
-                        if (index == parentModule.pipelineManager.getCurrentPipelineIndex()) {
+                        if (index == parentModule.pipelineManager.getRequestedIndex()) {
                             logger.debug("Skipping pipeline change, index " + index + " already active");
                             return;
                         }
@@ -181,6 +181,9 @@ public class VisionModuleChangeSubscriber extends DataChangeSubscriber {
                         parentModule.changePipelineType((Integer) newPropValue);
                         parentModule.saveAndBroadcastAll();
                         return;
+                    case "isDriverMode":
+                        parentModule.setDriverMode((Boolean) newPropValue);
+                        return;
                 }
 
                 // special case for camera settables

photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java

@@ -255,6 +255,8 @@ public class VisionSourceManager {
                 matches &= (physicalCamera.path.equals(savedConfig.path));
             }
 
+            matches &= (physicalCamera.cameraType == savedConfig.cameraType);
+
             return matches;
         };
     }
@@ -309,51 +311,64 @@ public class VisionSourceManager {
         ArrayList<CameraConfiguration> unloadedConfigs =
                 new ArrayList<CameraConfiguration>(loadedCamConfigs);
 
-        if (detectedCameraList.size() > 0 || unloadedConfigs.size() > 0) {
-            logger.info("Matching by usb port & name & USB VID/PID...");
-            cameraConfigurations.addAll(
-                    matchCamerasByStrategy(detectedCameraList, unloadedConfigs, true, true, true, false));
-        }
+        logger.info("Matching CSI cameras by port & base name...");
+        cameraConfigurations.addAll(
+                matchCamerasByStrategy(
+                        detectedCameraList,
+                        unloadedConfigs,
+                        new CameraMatchingOptions(false, false, true, true, CameraType.ZeroCopyPicam)));
+
+        logger.info("Matching USB cameras by usb port & name & USB VID/PID...");
+        cameraConfigurations.addAll(
+                matchCamerasByStrategy(
+                        detectedCameraList,
+                        unloadedConfigs,
+                        new CameraMatchingOptions(true, true, true, false, CameraType.UsbCamera)));
 
         // On windows, the v4l path is actually useful and tells us the port the camera is physically
         // connected to which is neat
         if (Platform.isWindows() && !matchCamerasOnlyByPath) {
-            if (detectedCameraList.size() > 0 || unloadedConfigs.size() > 0) {
-                logger.info("Matching by windows-path & USB VID/PID only...");
-                cameraConfigurations.addAll(
-                        matchCamerasByStrategy(detectedCameraList, unloadedConfigs, false, true, true, true));
-            }
+            logger.info("Matching USB cameras by windows-path & USB VID/PID only...");
+            cameraConfigurations.addAll(
+                    matchCamerasByStrategy(
+                            detectedCameraList,
+                            unloadedConfigs,
+                            new CameraMatchingOptions(false, true, true, true, CameraType.UsbCamera)));
         }
 
-        if (detectedCameraList.size() > 0 || unloadedConfigs.size() > 0) {
-            logger.info("Matching by usb port & USB VID/PID...");
-            cameraConfigurations.addAll(
-                    matchCamerasByStrategy(detectedCameraList, unloadedConfigs, true, true, false, false));
-        }
+        logger.info("Matching USB cameras by usb port & USB VID/PID...");
+        cameraConfigurations.addAll(
+                matchCamerasByStrategy(
+                        detectedCameraList,
+                        unloadedConfigs,
+                        new CameraMatchingOptions(true, true, false, false, CameraType.UsbCamera)));
 
         // Legacy migration -- VID/PID will be unset, so we have to try with our most relaxed strategy
         // at least once. We _should_ still have a valid USB path (assuming cameras have not moved), so
         // try that first, then fallback to base name only beloow
-        if (detectedCameraList.size() > 0 || unloadedConfigs.size() > 0) {
-            logger.info("Matching by base-name & usb port...");
-            cameraConfigurations.addAll(
-                    matchCamerasByStrategy(detectedCameraList, unloadedConfigs, true, false, true, false));
-        }
+        logger.info("Matching USB cameras by base-name & usb port...");
+        cameraConfigurations.addAll(
+                matchCamerasByStrategy(
+                        detectedCameraList,
+                        unloadedConfigs,
+                        new CameraMatchingOptions(true, false, true, false, CameraType.UsbCamera)));
 
         // handle disabling only-by-base-name matching
         if (!matchCamerasOnlyByPath) {
-            if (detectedCameraList.size() > 0 || unloadedConfigs.size() > 0) {
-                logger.info("Matching by base-name & USB VID/PID only...");
-                cameraConfigurations.addAll(
-                        matchCamerasByStrategy(detectedCameraList, unloadedConfigs, false, true, true, false));
-            }
+            logger.info("Matching USB cameras by base-name & USB VID/PID only...");
+            cameraConfigurations.addAll(
+                    matchCamerasByStrategy(
+                            detectedCameraList,
+                            unloadedConfigs,
+                            new CameraMatchingOptions(false, true, true, false, CameraType.UsbCamera)));
 
             // Legacy migration for if no USB VID/PID set
-            if (detectedCameraList.size() > 0 || unloadedConfigs.size() > 0) {
-                logger.info("Matching by base-name only...");
-                cameraConfigurations.addAll(
-                        matchCamerasByStrategy(detectedCameraList, unloadedConfigs, false, false, true, false));
-            }
+            logger.info("Matching USB cameras by base-name only...");
+            cameraConfigurations.addAll(
+                    matchCamerasByStrategy(
+                            detectedCameraList,
+                            unloadedConfigs,
+                            new CameraMatchingOptions(false, false, true, false, CameraType.UsbCamera)));
         } else logger.info("Skipping match by filepath/vid/pid, disabled by user");
 
         if (detectedCameraList.size() > 0) {
@@ -392,41 +407,46 @@ public class VisionSourceManager {
     private List<CameraConfiguration> matchCamerasByStrategy(
             List<CameraInfo> detectedCamInfos,
             List<CameraConfiguration> unloadedConfigs,
-            boolean checkUSBPath,
-            boolean checkVidPid,
-            boolean checkBaseName,
-            boolean checkPath) {
+            CameraMatchingOptions matchingOptions) {
         List<CameraConfiguration> ret = new ArrayList<CameraConfiguration>();
         List<CameraConfiguration> unloadedConfigsCopy =
                 new ArrayList<CameraConfiguration>(unloadedConfigs);
 
+        if (unloadedConfigsCopy.isEmpty()) return List.of();
+
+        logger.debug("Matching with options " + matchingOptions.toString());
+
         for (CameraConfiguration config : unloadedConfigsCopy) {
-            // Only run match path by id if the camera is not a CSI camera.
-            if (config.cameraType != CameraType.ZeroCopyPicam) {
+            // Only run match path by id if the camera type is allowed. This allows us to specify matching
+            // behavior per-camera-type
+            if (matchingOptions.allowedTypes.contains(config.cameraType)) {
                 logger.debug(
                         String.format(
-                                "Trying to find a match for loaded camera %s by strategy (path %s vid/pid %s basename %s path %s) with camera config: %s",
-                                config.baseName,
-                                checkUSBPath,
-                                checkVidPid,
-                                checkBaseName,
-                                checkPath,
-                                camCfgToString(config)));
+                                "Trying to find a match for loaded camera %s (%s) with camera config: %s",
+                                config.baseName, config.uniqueName, camCfgToString(config)));
 
                 // Get matcher and filter against it, picking out the first match
                 Predicate<CameraInfo> matches =
-                        getCameraMatcher(config, checkUSBPath, checkVidPid, checkBaseName, checkPath);
+                        getCameraMatcher(
+                                config,
+                                matchingOptions.checkUSBPath,
+                                matchingOptions.checkVidPid,
+                                matchingOptions.checkBaseName,
+                                matchingOptions.checkPath);
                 var cameraInfo = detectedCamInfos.stream().filter(matches).findFirst().orElse(null);
 
                 // If we actually matched a camera to a config, remove that camera from the list
                 // and add it to the output
                 if (cameraInfo != null) {
-                    logger.debug("Matched the config for " + config.baseName + " to a physical camera!");
+                    logger.debug(
+                            "Matched the config for "
+                                    + config.uniqueName
+                                    + " to the physical camera config above!");
                     ret.add(mergeInfoIntoConfig(config, cameraInfo));
                     detectedCamInfos.remove(cameraInfo);
                     unloadedConfigs.remove(config);
                 } else {
-                    logger.debug("No camera found for the config " + config.baseName);
+                    logger.debug("No camera found for the config " + config.uniqueName);
                 }
             }
         }
@@ -443,7 +463,10 @@ public class VisionSourceManager {
             List<CameraConfiguration> loadedConfigs) {
         List<CameraConfiguration> ret = new ArrayList<CameraConfiguration>();
         logger.debug(
-                "After matching loaded configs " + detectedCameraList.size() + " cameras were unmatched.");
+                "After matching loaded configs, these configs remained unmatched: "
+                        + detectedCameraList.stream()
+                                .map(n -> String.valueOf(n))
+                                .collect(Collectors.joining("-", "{", "}")));
         for (CameraInfo info : detectedCameraList) {
             // create new camera config for all new cameras
             String baseName = info.getBaseName();

photon-core/src/main/java/org/photonvision/vision/target/PotentialTarget.java

@@ -56,7 +56,7 @@ public class PotentialTarget implements Releasable {
     }
 
     public PotentialTarget(NeuralNetworkPipeResult det) {
-        this.shape = new CVShape(new Contour(det.box), ContourShape.Quadrilateral);
+        this.shape = new CVShape(new Contour(det.bbox), ContourShape.Quadrilateral);
         this.m_mainContour = this.shape.getContour();
         m_subContours = List.of();
         this.clsId = det.classIdx;

photon-core/src/test/java/org/photonvision/vision/processes/PipelineManagerTest.java

@@ -30,7 +30,7 @@ public class PipelineManagerTest {
     public void testUniqueName() {
         TestUtils.loadLibraries();
         PipelineManager manager =
-                new PipelineManager(new DriverModePipelineSettings(), List.of(), "meme_name");
+                new PipelineManager(new DriverModePipelineSettings(), List.of(), "meme_name", -1);
         manager.addPipeline(PipelineType.Reflective, "Another");
 
         // We now have ["New Pipeline", "Another"]

photon-core/src/test/java/org/photonvision/vision/processes/VisionSourceManagerTest.java

@@ -433,6 +433,73 @@ public class VisionSourceManagerTest {
         }
     }
 
+    @Test
+    public void testCSICameraMatching() {
+        Logger.setLevel(LogGroup.Camera, LogLevel.DEBUG);
+
+        // List of known cameras
+        var cameraInfos = new ArrayList<CameraInfo>();
+
+        var inst = new VisionSourceManager();
+        ConfigManager.getInstance().clearConfig();
+        ConfigManager.getInstance().load();
+        ConfigManager.getInstance().getConfig().getNetworkConfig().matchCamerasOnlyByPath = false;
+
+        CameraInfo info1 =
+                new CameraInfo(
+                        -1,
+                        "/base/soc/i2c0mux/i2c@0/ov9281@60",
+                        "OV9281", // Typically rp1-cfe for unit test changed to CSICAM-DEV
+                        new String[] {},
+                        -1,
+                        -1,
+                        CameraType.ZeroCopyPicam);
+
+        CameraInfo info2 =
+                new CameraInfo(
+                        -1,
+                        "/base/soc/i2c0mux/i2c@1/ov9281@60",
+                        "OV9281", // Typically rp1-cfe for unit test changed to CSICAM-DEV
+                        new String[] {},
+                        -1,
+                        -1,
+                        CameraType.ZeroCopyPicam);
+
+        var camera1_saved_config =
+                new CameraConfiguration(
+                        "OV9281", "OV9281", "test-1", "/base/soc/i2c0mux/i2c@0/ov9281@60", new String[0]);
+        camera1_saved_config.cameraType = CameraType.ZeroCopyPicam;
+        camera1_saved_config.usbVID = -1;
+        camera1_saved_config.usbPID = -1;
+
+        var camera2_saved_config =
+                new CameraConfiguration(
+                        "OV9281", "OV9281 (1)", "test-2", "/base/soc/i2c0mux/i2c@1/ov9281@60", new String[0]);
+        camera2_saved_config.usbVID = -1;
+        camera2_saved_config.usbPID = -1;
+        camera2_saved_config.cameraType = CameraType.ZeroCopyPicam;
+
+        cameraInfos.add(info1);
+        cameraInfos.add(info2);
+
+        // Try matching with both cameras being "known"
+        inst.registerLoadedConfigs(camera1_saved_config, camera2_saved_config);
+        var ret1 = inst.tryMatchCamImpl(cameraInfos);
+
+        // Our cameras should be "known"
+        assertTrue(inst.knownCameras.contains(info1));
+        assertTrue(inst.knownCameras.contains(info2));
+        assertEquals(2, inst.knownCameras.size());
+        assertEquals(2, ret1.size());
+
+        // Exactly one camera should have the path we put in
+        for (int i = 0; i < cameraInfos.size(); i++) {
+            var testPath = cameraInfos.get(i).path;
+            assertEquals(
+                    1, ret1.stream().filter(it -> testPath.equals(it.cameraConfiguration.path)).count());
+        }
+    }
+
     @Test
     public void testIdenticalCameras() {
         Logger.setLevel(LogGroup.Camera, LogLevel.DEBUG);

photon-lib/src/generate/native/cpp/PhotonVersion.cpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <string.h>
+#include <regex>
+
+/*
+ * Autogenerated file! Do not manually edit this file. This version is
+ * regenerated any time the publish task is run, or when this file is deleted.
+ */
+
+static const char* dev_ = "dev";
+
+namespace photon {
+  namespace PhotonVersion {
+    const char* versionString = "dev-v2024.3.1-22-g38e2fdef";
+    const char* buildDate = "2024-5-29 10:05:11";
+    const bool isRelease = strncmp(dev_, versionString, strlen(dev_)) != 0;
+  }
+}

photon-lib/src/main/native/include/photon/simulation/SimCameraProperties.h

@@ -96,12 +96,12 @@ class SimCameraProperties {
                                       units::radian_t{std::numbers::pi / 2.0}})
                                      .Radians()}},
         frc::Translation3d{
-            1_m, frc::Rotation3d{0_rad,
-                                 (GetPixelPitch(0) +
-                                  frc::Rotation2d{
-                                      units::radian_t{std::numbers::pi / 2.0}})
-                                     .Radians(),
-                                 0_rad}},
+            1_m,
+            frc::Rotation3d{0_rad,
+                            (GetPixelPitch(0) + frc::Rotation2d{units::radian_t{
+                                                    std::numbers::pi / 2.0}})
+                                .Radians(),
+                            0_rad}},
         frc::Translation3d{
             1_m, frc::Rotation3d{0_rad,
                                  (GetPixelPitch(height) +

photon-server/src/main/java/org/photonvision/Main.java

@@ -35,6 +35,7 @@ import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.LogLevel;
 import org.photonvision.common.logging.Logger;
+import org.photonvision.common.logging.PvCSCoreLogger;
 import org.photonvision.common.networking.NetworkManager;
 import org.photonvision.common.util.TestUtils;
 import org.photonvision.common.util.numbers.IntegerCouple;
@@ -65,6 +66,7 @@ public class Main {
     private static final boolean isRelease = PhotonVersion.isRelease;
 
     private static boolean isTestMode = false;
+    private static boolean isSmoketest = false;
     private static Path testModeFolder = null;
     private static boolean printDebugLogs;
 
@@ -90,6 +92,11 @@ public class Main {
                 "clear-config",
                 false,
                 "Clears PhotonVision pipeline and networking settings. Preserves log files");
+        options.addOption(
+                "s",
+                "smoketest",
+                false,
+                "Exit Photon after loading native libraries and camera configs, but before starting up camera runners");
 
         CommandLineParser parser = new DefaultParser();
         CommandLine cmd = parser.parse(options, args);
@@ -127,6 +134,10 @@ public class Main {
             if (cmd.hasOption("clear-config")) {
                 ConfigManager.getInstance().clearConfig();
             }
+
+            if (cmd.hasOption("smoketest")) {
+                isSmoketest = true;
+            }
         }
         return true;
     }
@@ -266,7 +277,7 @@ public class Main {
 
         CameraConfiguration camConf2024 =
                 ConfigManager.getInstance().getConfig().getCameraConfigurations().get("WPI2024");
-        if (camConf2024 == null || true) {
+        if (camConf2024 == null) {
             camConf2024 =
                     new CameraConfiguration(
                             "WPI2024",
@@ -337,11 +348,17 @@ public class Main {
 
     public static void main(String[] args) {
         try {
-            TestUtils.loadLibraries();
-            logger.info("Native libraries loaded.");
+            boolean success = TestUtils.loadLibraries();
+
+            if (!success) {
+                logger.error("Failed to load native libraries! Giving up :(");
+                System.exit(1);
+            }
         } catch (Exception e) {
             logger.error("Failed to load native libraries!", e);
+            System.exit(1);
         }
+        logger.info("Native libraries loaded.");
 
         try {
             if (Platform.isRaspberryPi()) {
@@ -393,6 +410,8 @@ public class Main {
                         + Platform.getPlatformName()
                         + (Platform.isRaspberryPi() ? (" (Pi " + PiVersion.getPiVersion() + ")") : ""));
 
+        PvCSCoreLogger.getInstance();
+
         logger.debug("Loading ConfigManager...");
         ConfigManager.getInstance().load(); // init config manager
         ConfigManager.getInstance().requestSave();
@@ -412,6 +431,11 @@ public class Main {
         NeuralNetworkModelManager.getInstance()
                 .initialize(ConfigManager.getInstance().getModelsDirectory());
 
+        if (isSmoketest) {
+            logger.info("PhotonVision base functionality loaded -- smoketest complete");
+            System.exit(0);
+        }
+
         if (!isTestMode) {
             logger.debug("Loading VisionSourceManager...");
             VisionSourceManager.getInstance()

photon-server/src/main/java/org/photonvision/server/DataSocketHandler.java

@@ -131,18 +131,16 @@ public class DataSocketHandler {
                         case SMT_DRIVERMODE:
                             {
                                 // TODO: what is this event?
-                                var data = (HashMap<String, Object>) entryValue;
-                                var dmExpEvent =
-                                        new IncomingWebSocketEvent<Integer>(
-                                                DataChangeDestination.DCD_ACTIVEMODULE, "driverExposure", data);
-                                var dmBrightEvent =
-                                        new IncomingWebSocketEvent<Integer>(
-                                                DataChangeDestination.DCD_ACTIVEMODULE, "driverBrightness", data);
+                                var data = (Boolean) entryValue;
                                 var dmIsDriverEvent =
                                         new IncomingWebSocketEvent<Boolean>(
-                                                DataChangeDestination.DCD_ACTIVEMODULE, "isDriver", data);
+                                                DataChangeDestination.DCD_ACTIVEMODULE,
+                                                "isDriverMode",
+                                                data,
+                                                cameraIndex,
+                                                context);
 
-                                dcService.publishEvents(dmExpEvent, dmBrightEvent, dmIsDriverEvent);
+                                dcService.publishEvents(dmIsDriverEvent);
                                 break;
                             }
                         case SMT_CHANGECAMERANAME:

photonlib-cpp-examples/aimandrange/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "cpp"
     id "google-test-test-suite"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 
     id "com.dorongold.task-tree" version "2.1.0"
 }

photonlib-cpp-examples/aimattarget/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "cpp"
     id "google-test-test-suite"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 
     id "com.dorongold.task-tree" version "2.1.0"
 }

photonlib-cpp-examples/apriltagExample/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "cpp"
     id "google-test-test-suite"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 
     id "com.dorongold.task-tree" version "2.1.0"
 }

photonlib-cpp-examples/build.gradle

@@ -6,7 +6,7 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
+        maven { url = "https://maven.photonvision.org/releases" }
     }
 }
 

photonlib-cpp-examples/getinrange/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "cpp"
     id "google-test-test-suite"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 
     id "com.dorongold.task-tree" version "2.1.0"
 }

photonlib-cpp-examples/swervedriveposeestsim/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "cpp"
     id "google-test-test-suite"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 
     id "com.dorongold.task-tree" version "2.1.0"
 }
@@ -12,8 +12,8 @@ repositories {
 }
 
 wpi.maven.useDevelopment = true
-wpi.versions.wpilibVersion = "2024.3.1"
-wpi.versions.wpimathVersion = "2024.3.1"
+wpi.versions.wpilibVersion = "2024.3.2"
+wpi.versions.wpimathVersion = "2024.3.2"
 
 apply from: "${rootDir}/../shared/examples_common.gradle"
 

photonlib-java-examples/aimandrange/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "java"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 }
 
 sourceCompatibility = JavaVersion.VERSION_11

photonlib-java-examples/aimattarget/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "java"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 }
 
 sourceCompatibility = JavaVersion.VERSION_11

photonlib-java-examples/build.gradle

@@ -8,7 +8,7 @@ allprojects {
     repositories {
         mavenCentral()
         mavenLocal()
-        maven { url = "https://maven.photonvision.org/repository/internal/" }
+        maven { url = "https://maven.photonvision.org/releases" }
     }
 }
 

photonlib-java-examples/getinrange/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "java"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 }
 
 sourceCompatibility = JavaVersion.VERSION_11

photonlib-java-examples/simaimandrange/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "java"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 }
 
 sourceCompatibility = JavaVersion.VERSION_11

photonlib-java-examples/swervedriveposeestsim/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "java"
-    id "edu.wpi.first.GradleRIO" version "2024.3.1"
+    id "edu.wpi.first.GradleRIO" version "2024.3.2"
 }
 
 sourceCompatibility = JavaVersion.VERSION_11
@@ -11,8 +11,8 @@ apply from: "${rootDir}/../shared/examples_common.gradle"
 def ROBOT_MAIN_CLASS = "frc.robot.Main"
 
 wpi.maven.useDevelopment = true
-wpi.versions.wpilibVersion = "2024.3.1"
-wpi.versions.wpimathVersion = "2024.3.1"
+wpi.versions.wpilibVersion = "2024.3.2"
+wpi.versions.wpimathVersion = "2024.3.2"
 
 
 // Define my targets (RoboRIO) and artifacts (deployable files)

shared/javacommon.gradle

@@ -79,7 +79,7 @@ publishing {
 
     repositories {
         maven {
-            url ('https://maven.photonvision.org/repository/' + (isDev ? 'snapshots' : 'internal'))
+            url(photonMavenURL)
             credentials {
                 username 'ghactions'
                 password System.getenv("ARTIFACTORY_API_KEY")

shared/javacpp/publish.gradle

@@ -70,7 +70,7 @@ model {
 
         repositories {
             maven {
-                url ('https://maven.photonvision.org/repository/' + (isDev ? 'snapshots' : 'internal'))
+                url(photonMavenURL)
                 credentials {
                     username 'ghactions'
                     password System.getenv("ARTIFACTORY_API_KEY")